Skip to content

Modern Docker Management, Designed for Everyone

refactor(system): use composite literals, drop DI nil checks and mappers

Kyle Mendellcdfe49e8cbe0
diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.go
index cabd3c168..2c8b8dfa6 100644
--- a/backend/internal/backup/service.go
+++ b/backend/internal/backup/service.go
@@ -161,9 +161,6 @@ func NewEngine(admission *runs.Admission, imageService *image.ImageService) *Eng
// TryAcquireRun takes the admission lease for one backup resource, reporting false while another run holds it.
func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {
- if e == nil || e.admission == nil {
- return nil, false, errors.New("backup engine is unavailable")
- }
return e.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: scope, ID: id})
}
@@ -307,12 +304,7 @@ func qualifySnapshotListing(files []string, prefix string) []string {
return qualified
}
for index, file := range qualified {
- directory := strings.HasSuffix(file, "/")
- relative := strings.TrimPrefix(strings.TrimPrefix(file, "./"), "/")
- qualified[index] = path.Join(prefix, relative)
- if directory {
- qualified[index] += "/"
- }
+ qualified[index] = path.Join(prefix, strings.TrimPrefix(strings.TrimPrefix(file, "./"), "/")) + kit.Ternary(strings.HasSuffix(file, "/"), "/", "")
}
return qualified
}
@@ -393,11 +385,7 @@ func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Clien
}
var found Snapshot
for _, snapshot := range snapshots {
- match := snapshotID != "" && snapshot.ID == snapshotID
- if snapshotID == "" {
- match = slices.Contains(snapshot.Tags, RunSnapshotTag(runID))
- }
- if !match {
+ if !kit.Ternary(snapshotID == "", slices.Contains(snapshot.Tags, RunSnapshotTag(runID)), snapshot.ID == snapshotID) {
continue
}
if found.ID != "" {
@@ -558,12 +546,19 @@ func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, fro
defer func() {
_, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), temporaryVolume, client.VolumeRemoveOptions{Force: true})
}()
- copyMount := mount.Mount{Type: mount.TypeVolume, Source: temporaryVolume, Target: "/volume"}
- if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, copyMount, RestoreOptions{DeleteExtra: true}); err != nil {
+ if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, mount.Mount{
+ Type: mount.TypeVolume,
+ Source: temporaryVolume,
+ Target: "/volume",
+ }, RestoreOptions{DeleteExtra: true}); err != nil {
return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)
}
- copyMount.ReadOnly = true
- snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(copyMount, tags...))
+ snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(mount.Mount{
+ Type: mount.TypeVolume,
+ Source: temporaryVolume,
+ Target: "/volume",
+ ReadOnly: true,
+ }, tags...))
if err != nil {
return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)
}
@@ -582,22 +577,26 @@ func (e *Engine) run(ctx context.Context, dockerClient *client.Client, repositor
// runContainer runs one Rustic command; callers hold the repository lock.
func (e *Engine) runContainer(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err != nil {
- if e.imageService == nil {
- return "", errors.New("image service is unavailable")
- }
if pullErr := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); pullErr != nil {
return "", fmt.Errorf("failed to pull official Rustic image: %w", pullErr)
}
}
- var networkMode container.NetworkMode
- arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
- if err != nil || arcane == nil || arcane.ID == "" {
- slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)
- } else {
- networkMode = container.NetworkMode("container:" + arcane.ID)
- }
- mounts := append(slices.Clone(repository.Mounts), extraMounts...)
- return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, networkMode)
+ return rustic.Run(
+ ctx,
+ dockerClient,
+ password,
+ command,
+ repository.Environment,
+ append(slices.Clone(repository.Mounts), extraMounts...),
+ func() container.NetworkMode {
+ arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ if err != nil || kit.FromPtr(arcane).ID == "" {
+ slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)
+ return ""
+ }
+ return container.NetworkMode("container:" + arcane.ID)
+ }(),
+ )
}
var (
@@ -719,8 +718,13 @@ func (s *RecoveryKeyStore) save(ctx context.Context, id, recoveryKey string) err
func AcceptedTarget(backupID string) scheduler.TargetOutcome {
data, _ := json.Marshal(struct {
BackupID string `json:"backupId"`
- }{backupID})
- return scheduler.TargetOutcome{ResourceType: "backup", ID: "accepted:" + backupID, Status: scheduler.Queued, RecoveryData: data}
+ }{BackupID: backupID})
+ return scheduler.TargetOutcome{
+ ResourceType: "backup",
+ ID: "accepted:" + backupID,
+ Status: scheduler.Queued,
+ RecoveryData: data,
+ }
}
// AcceptedBackupID returns the backup a manual backup workflow's run owns, from its AcceptedTarget.
@@ -736,9 +740,6 @@ func AcceptedBackupID(run scheduler.Run) string {
// lockRepository serializes commands against one repository and returns its unlock func.
func (e *Engine) lockRepository(ctx context.Context, id string) (func(), error) {
- if e == nil {
- return nil, errors.New("backup engine is unavailable")
- }
if strings.TrimSpace(id) == "" {
return nil, errors.New("backup repository ID is required")
}
@@ -815,8 +816,6 @@ func (r PolicyReconciliation[P, U]) Run(ctx context.Context, updates []U) error
switch {
case r.Persist != nil:
persistErr = r.Persist(ctx, policies)
- case r.DB == nil:
- persistErr = errors.New("backup policy database is unavailable")
default:
persistErr = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for i := range policies {
@@ -850,27 +849,30 @@ func ValidatePolicyUpdate(ctx context.Context, domain string, update backup.Upda
if err != nil {
return update, err
}
- update.Schedule = normalized
if update.RetentionCount < 0 || update.RetentionCount > 3650 {
return update, errors.New("retentionCount must be between 0 and 3650")
}
if !update.LocalEnabled && !update.S3Enabled {
return update, fmt.Errorf("select at least one %s backup destination", domain)
}
+ if update.S3Enabled && strings.TrimSpace(update.S3DestinationID) == "" {
+ return update, fmt.Errorf("select an S3 destination for %s backups", domain)
+ }
if update.S3Enabled {
- if strings.TrimSpace(update.S3DestinationID) == "" {
- return update, fmt.Errorf("select an S3 destination for %s backups", domain)
- }
- if s3Service == nil {
- return update, errors.New("S3 backup destinations are unavailable")
- }
if _, configurationErr := s3Service.Configuration(ctx, update.S3DestinationID); configurationErr != nil {
return update, fmt.Errorf("select a valid S3 destination for %s backups", domain)
}
- } else {
- update.S3DestinationID = ""
}
- return update, nil
+ return backup.UpdateBackupPolicy{
+ ID: update.ID,
+ Enabled: update.Enabled,
+ Schedule: normalized,
+ RetentionCount: update.RetentionCount,
+ StopContainers: update.StopContainers,
+ LocalEnabled: update.LocalEnabled,
+ S3Enabled: update.S3Enabled,
+ S3DestinationID: kit.Ternary(update.S3Enabled, update.S3DestinationID, ""),
+ }, nil
}
var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")
@@ -883,7 +885,7 @@ func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationSe
checked := make(map[string]*backup.RepositoryObservation)
deadline := time.Now().Add(10 * time.Second)
return func(destinationID, root, snapshotID string) *bool {
- if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {
+ if root == "" || destinationID == "" || snapshotID == "" {
return nil
}
key := destinationID + ":" + root + ":" + snapshotID
@@ -913,9 +915,6 @@ func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationSe
// CheckRemoteRepository probes the destination's repository config within a bounded budget.
// Transport and credential failures are returned as errors so callers never forget snapshots blindly.
func CheckRemoteRepository(ctx context.Context, destinations *s3.S3DestinationService, destinationID, root string) (backup.RepositoryObservation, error) {
- if destinations == nil {
- return backup.RepositoryObservation{}, errors.New("S3 destinations are unavailable")
- }
configuration, err := destinations.Configuration(ctx, destinationID)
if err != nil {
return backup.RepositoryObservation{}, err
@@ -932,15 +931,11 @@ func DisableMissingRemote(remoteErr error, localEnabled bool, s3Enabled, enabled
if !errors.Is(remoteErr, ErrRemoteRepositoryMissing) {
return false, nil
}
- field, flag := "enabled", enabled
- if localEnabled {
- field, flag = "s3_enabled", s3Enabled
- }
- saved, err := save(field)
+ saved, err := save(kit.Ternary(localEnabled, "s3_enabled", "enabled"))
if err != nil || !saved {
return false, err
}
- *flag = false
+ *kit.Ternary(localEnabled, s3Enabled, enabled) = false
return true, nil
}
diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.go
index 5519b880e..1bde751ba 100644
--- a/backend/internal/s3/handler.go
+++ b/backend/internal/s3/handler.go
@@ -168,15 +168,11 @@ func (h *s3DestinationHandler) test(ctx context.Context, input *testS3Destinatio
}
func (h *s3DestinationHandler) triggerRemoteSync(ctx context.Context, reason string) {
- if h.syncRemoteDestinations == nil {
- return
- }
- detachedCtx := context.WithoutCancel(ctx)
go func(syncCtx context.Context, syncReason string) {
if err := h.syncRemoteDestinations(syncCtx); err != nil {
slog.WarnContext(syncCtx, s3RemoteSyncFailureMessage, "reason", syncReason, "error", err.Error())
}
- }(detachedCtx, reason)
+ }(context.WithoutCancel(ctx), reason)
}
func (h *s3DestinationHandler) testConfiguration(ctx context.Context, input *testS3DestinationConfigurationInput) (*handlerutil.Out[base.MessageResponse], error) {
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
index 9812cbf27..6a9fa4c30 100644
--- a/backend/internal/s3/service.go
+++ b/backend/internal/s3/service.go
@@ -1,6 +1,7 @@
package s3
import (
+ "cmp"
"context"
"errors"
"fmt"
@@ -8,8 +9,8 @@ import (
"strings"
"github.com/getarcaneapp/arcane/types/v2/backup"
- "github.com/samber/mo"
"go.getarcane.app/kit/normalization"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
@@ -22,9 +23,9 @@ import (
var (
ErrS3DestinationNotFound = errors.New("S3 destination not found")
ErrS3DestinationInUse = errors.New("S3 destination is in use")
- // errS3SecretRequiredInternal enforces the fresh-secret rule: a stored
+ // errS3SecretRequired enforces the fresh-secret rule: a stored
// secret is never combined with changed connection or trust settings.
- errS3SecretRequiredInternal = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style")
+ errS3SecretRequired = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style")
)
type S3DestinationService struct {
@@ -39,7 +40,7 @@ func NewS3DestinationService(db *database.DB, checkRemoteReferences func(context
return &S3DestinationService{db: db, checkRemoteReferences: checkRemoteReferences}
}
-func destinationConfigurationInternal(id string, input backup.CreateS3Destination) s3.Configuration {
+func destinationConfiguration(id string, input backup.CreateS3Destination) s3.Configuration {
return s3.Configuration{
ID: id,
Name: input.Name,
@@ -54,14 +55,6 @@ func destinationConfigurationInternal(id string, input backup.CreateS3Destinatio
}.Normalized()
}
-func s3DestinationsToDTOsInternal(destinations []S3Destination) []backup.S3Destination {
- result := make([]backup.S3Destination, len(destinations))
- for i := range destinations {
- result[i] = destinations[i].ToDTO()
- }
- return result
-}
-
func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params pagination.QueryParams) ([]backup.S3Destination, pagination.Response, error) {
var destinations []S3Destination
query := s.db.WithContext(ctx).Model(&S3Destination{})
@@ -73,7 +66,11 @@ func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params pa
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list S3 destinations: %w", err)
}
- return s3DestinationsToDTOsInternal(destinations), response, nil
+ result := make([]backup.S3Destination, len(destinations))
+ for i := range destinations {
+ result[i] = destinations[i].ToDTO()
+ }
+ return result, response, nil
}
func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]backup.S3Destination, error) {
@@ -81,15 +78,16 @@ func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]bac
if err := s.db.WithContext(ctx).Order("name ASC").Find(&destinations).Error; err != nil {
return nil, fmt.Errorf("failed to list S3 destinations: %w", err)
}
- return s3DestinationsToDTOsInternal(destinations), nil
+ result := make([]backup.S3Destination, len(destinations))
+ for i := range destinations {
+ result[i] = destinations[i].ToDTO()
+ }
+ return result, nil
}
// DestinationsByID indexes destinations to decorate backup records, such as with their names. Decoration never fails
-// its caller, so a missing service or failed listing yields an empty index.
+// its caller, so a failed listing yields an empty index.
func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]backup.S3Destination {
- if s == nil {
- return nil
- }
destinations, err := s.ListAllS3Destinations(ctx)
if err != nil {
slog.WarnContext(ctx, "could not list S3 destinations", "error", err)
@@ -102,7 +100,7 @@ func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]
return indexed
}
-func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {
+func (s *S3DestinationService) getS3DestinationModel(ctx context.Context, id string) (*S3Destination, error) {
var destination S3Destination
if err := s.db.WithContext(ctx).Where("id = ?", strings.TrimSpace(id)).First(&destination).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -114,7 +112,7 @@ func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context
}
func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string) (*backup.S3Destination, error) {
- destination, err := s.getS3DestinationModelInternal(ctx, id)
+ destination, err := s.getS3DestinationModel(ctx, id)
if err != nil {
return nil, err
}
@@ -122,26 +120,11 @@ func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string)
return &dto, nil
}
-// applyS3ConfigurationInternal copies configuration onto destination and
-// reports whether any field actually changed.
-func applyS3ConfigurationInternal(destination *S3Destination, configuration s3.Configuration, encryptedSecret string) bool {
- changed := utils.ApplyChanged(&destination.Name, mo.Some(configuration.Name))
- changed = utils.ApplyChanged(&destination.Endpoint, mo.Some(configuration.Endpoint)) || changed
- changed = utils.ApplyChanged(&destination.Bucket, mo.Some(configuration.Bucket)) || changed
- changed = utils.ApplyChanged(&destination.Region, mo.Some(configuration.Region)) || changed
- changed = utils.ApplyChanged(&destination.AccessKeyID, mo.Some(configuration.AccessKeyID)) || changed
- changed = utils.ApplyChanged(&destination.SecretAccessKey, mo.Some(encryptedSecret)) || changed
- changed = utils.ApplyChanged(&destination.Prefix, mo.Some(configuration.Prefix)) || changed
- changed = utils.ApplyChanged(&destination.UseSSL, mo.Some(configuration.UseSSL)) || changed
- changed = utils.ApplyChanged(&destination.ForcePathStyle, mo.Some(configuration.ForcePathStyle)) || changed
- return changed
-}
-
func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input backup.CreateS3Destination) (*backup.S3Destination, error) {
if err := normalization.Normalize(&input); err != nil {
return nil, err
}
- configuration := destinationConfigurationInternal("", input)
+ configuration := destinationConfiguration("", input)
if err := configuration.Validate(true); err != nil {
return nil, err
}
@@ -149,8 +132,17 @@ func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input ba
if err != nil {
return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)
}
- destination := &S3Destination{}
- applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
+ destination := &S3Destination{
+ Name: configuration.Name,
+ Endpoint: configuration.Endpoint,
+ Bucket: configuration.Bucket,
+ Region: configuration.Region,
+ AccessKeyID: configuration.AccessKeyID,
+ SecretAccessKey: encryptedSecret,
+ Prefix: configuration.Prefix,
+ UseSSL: configuration.UseSSL,
+ ForcePathStyle: configuration.ForcePathStyle,
+ }
if createDestinationErr := s.db.WithContext(ctx).Create(destination).Error; createDestinationErr != nil {
return nil, fmt.Errorf("failed to create S3 destination: %w", createDestinationErr)
}
@@ -158,45 +150,54 @@ func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input ba
return &dto, nil
}
-// storedConfigurationInternal returns the persisted configuration without the
-// decrypted secret, for comparing connection fields against an update.
-func storedConfigurationInternal(destination *S3Destination) s3.Configuration {
- return s3.Configuration{
- ID: destination.ID,
- Name: destination.Name,
- Endpoint: destination.Endpoint,
- Bucket: destination.Bucket,
- Region: destination.Region,
- AccessKeyID: destination.AccessKeyID,
- Prefix: destination.Prefix,
- UseSSL: destination.UseSSL,
- ForcePathStyle: destination.ForcePathStyle,
- }.Normalized()
-}
-
func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id string, input backup.UpdateS3Destination) (*backup.S3Destination, error) {
if err := normalization.Normalize(&input); err != nil {
return nil, err
}
- configuration := destinationConfigurationInternal("", input)
+ configuration := destinationConfiguration("", input)
if err := configuration.Validate(false); err != nil {
return nil, err
}
- destination, err := s.getS3DestinationModelInternal(ctx, id)
+ destination, err := s.getS3DestinationModel(ctx, id)
if err != nil {
return nil, err
}
- if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(storedConfigurationInternal(destination), configuration) {
- return nil, errS3SecretRequiredInternal
+ // The persisted configuration without the
+ // decrypted secret, for comparing connection fields against an update.
+ if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(s3.Configuration{
+ ID: destination.ID,
+ Name: destination.Name,
+ Endpoint: destination.Endpoint,
+ Bucket: destination.Bucket,
+ Region: destination.Region,
+ AccessKeyID: destination.AccessKeyID,
+ Prefix: destination.Prefix,
+ UseSSL: destination.UseSSL,
+ ForcePathStyle: destination.ForcePathStyle,
+ }.Normalized(), configuration) {
+ return nil, errS3SecretRequired
}
- encryptedSecret := destination.SecretAccessKey
- if configuration.SecretAccessKey != "" {
- encryptedSecret, err = crypto.Encrypt(configuration.SecretAccessKey)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)
+ encryptedSecret, err := func() (string, error) {
+ if configuration.SecretAccessKey == "" {
+ return destination.SecretAccessKey, nil
}
+ return crypto.Encrypt(configuration.SecretAccessKey)
+ }()
+ if err != nil {
+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)
+ }
+ *destination = S3Destination{ //exhaustruct:enforce
+ BaseModel: destination.BaseModel,
+ Name: configuration.Name,
+ Endpoint: configuration.Endpoint,
+ Bucket: configuration.Bucket,
+ Region: configuration.Region,
+ AccessKeyID: configuration.AccessKeyID,
+ SecretAccessKey: encryptedSecret,
+ Prefix: configuration.Prefix,
+ UseSSL: configuration.UseSSL,
+ ForcePathStyle: configuration.ForcePathStyle,
}
- applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
if updateDestinationErr := s.db.WithContext(ctx).Save(destination).Error; updateDestinationErr != nil {
return nil, fmt.Errorf("failed to update S3 destination: %w", updateDestinationErr)
}
@@ -205,7 +206,7 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strin
}
func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id string) error {
- destination, err := s.getS3DestinationModelInternal(ctx, id)
+ destination, err := s.getS3DestinationModel(ctx, id)
if err != nil {
return err
}
@@ -216,10 +217,8 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strin
if inUse {
return fmt.Errorf("%w: remove it from backup configurations and retained backups first", ErrS3DestinationInUse)
}
- if s.checkRemoteReferences != nil {
- if checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {
- return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())
- }
+ if checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {
+ return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())
}
if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {
return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)
@@ -266,41 +265,56 @@ func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinati
if err := s.db.WithContext(ctx).Find(&existing).Error; err != nil {
return fmt.Errorf("failed to load existing S3 destinations: %w", err)
}
- existingByID := make(map[string]*S3Destination, len(existing))
- for i := range existing {
- existingByID[existing[i].ID] = &existing[i]
+ existingByID := make(map[string]S3Destination, len(existing))
+ for _, destination := range existing {
+ existingByID[destination.ID] = destination
}
syncedIDs := make(map[string]struct{}, len(destinations))
for _, item := range destinations {
- configuration := destinationConfigurationInternal(item.ID, item.CreateS3Destination)
+ configuration := destinationConfiguration(item.ID, item.CreateS3Destination)
if configuration.ID == "" {
return errors.New("S3 destination ID is required")
}
if err := configuration.Validate(true); err != nil {
return fmt.Errorf("invalid S3 destination %s: %w", configuration.ID, err)
}
- destination, exists := existingByID[configuration.ID]
- if !exists {
- destination = &S3Destination{ID: configuration.ID}
- }
+ current, exists := existingByID[configuration.ID]
// The manager resends the full set every health tick; unchanged rows
// keep their ciphertext and skip the write.
- secretChanged, err := utils.ApplyEncrypted(&destination.SecretAccessKey, configuration.SecretAccessKey)
+ encryptedSecret := current.SecretAccessKey
+ secretChanged, err := utils.ApplyEncrypted(&encryptedSecret, configuration.SecretAccessKey)
if err != nil {
return fmt.Errorf("failed to apply S3 secret access key for %s: %w", configuration.ID, err)
}
- changed := applyS3ConfigurationInternal(destination, configuration, destination.SecretAccessKey) || secretChanged || !exists
- if !item.CreatedAt.IsZero() && !destination.CreatedAt.Equal(item.CreatedAt) {
- destination.CreatedAt = item.CreatedAt
- changed = true
- }
- if item.UpdatedAt != nil && (destination.UpdatedAt == nil || !destination.UpdatedAt.Equal(*item.UpdatedAt)) {
- destination.UpdatedAt = item.UpdatedAt
- changed = true
- }
+ changed := !exists || secretChanged ||
+ current.Name != configuration.Name ||
+ current.Endpoint != configuration.Endpoint ||
+ current.Bucket != configuration.Bucket ||
+ current.Region != configuration.Region ||
+ current.AccessKeyID != configuration.AccessKeyID ||
+ current.Prefix != configuration.Prefix ||
+ current.UseSSL != configuration.UseSSL ||
+ current.ForcePathStyle != configuration.ForcePathStyle ||
+ (!item.CreatedAt.IsZero() && !current.CreatedAt.Equal(item.CreatedAt)) ||
+ (item.UpdatedAt != nil && !kit.FromPtr(current.UpdatedAt).Equal(*item.UpdatedAt))
if changed {
- if syncDestinationErr := s.db.WithContext(ctx).Save(destination).Error; syncDestinationErr != nil {
+ if syncDestinationErr := s.db.WithContext(ctx).Save(&S3Destination{ //exhaustruct:enforce
+ BaseModel: database.BaseModel{
+ ID: configuration.ID,
+ CreatedAt: kit.Ternary(item.CreatedAt.IsZero(), current.CreatedAt, item.CreatedAt),
+ UpdatedAt: kit.Ternary(item.UpdatedAt != nil, item.UpdatedAt, current.UpdatedAt),
+ },
+ Name: configuration.Name,
+ Endpoint: configuration.Endpoint,
+ Bucket: configuration.Bucket,
+ Region: configuration.Region,
+ AccessKeyID: configuration.AccessKeyID,
+ SecretAccessKey: encryptedSecret,
+ Prefix: configuration.Prefix,
+ UseSSL: configuration.UseSSL,
+ ForcePathStyle: configuration.ForcePathStyle,
+ }).Error; syncDestinationErr != nil {
return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, syncDestinationErr)
}
}
@@ -323,7 +337,7 @@ func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinati
// Configuration returns a destination's decrypted runtime configuration for backup domains.
func (s *S3DestinationService) Configuration(ctx context.Context, id string) (s3.Configuration, error) {
- destination, err := s.getS3DestinationModelInternal(ctx, id)
+ destination, err := s.getS3DestinationModel(ctx, id)
if err != nil {
return s3.Configuration{}, err
}
@@ -350,28 +364,35 @@ func (s *S3DestinationService) TestS3Destination(ctx context.Context, id string,
if err != nil {
return err
}
- if input != nil {
- if normalizeErr := normalization.Normalize(input); normalizeErr != nil {
- return normalizeErr
- }
- updated := destinationConfigurationInternal("", *input)
- updated.ID = configuration.ID
- if updated.SecretAccessKey == "" {
- // The stored secret may only sign requests against the stored
- // connection settings; any change requires a fresh secret.
- if !s3.ConnectionFieldsEqual(configuration, updated) {
- return errS3SecretRequiredInternal
- }
- updated.SecretAccessKey = configuration.SecretAccessKey
- }
- configuration = updated
- }
- return s3.TestConnection(ctx, configuration)
+ if input == nil {
+ return s3.TestConnection(ctx, configuration)
+ }
+ if normalizeErr := normalization.Normalize(input); normalizeErr != nil {
+ return normalizeErr
+ }
+ updated := destinationConfiguration(configuration.ID, *input)
+ // The stored secret may only sign requests against the stored
+ // connection settings; any change requires a fresh secret.
+ if updated.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(configuration, updated) {
+ return errS3SecretRequired
+ }
+ return s3.TestConnection(ctx, s3.Configuration{
+ ID: updated.ID,
+ Name: updated.Name,
+ Endpoint: updated.Endpoint,
+ Bucket: updated.Bucket,
+ Region: updated.Region,
+ AccessKeyID: updated.AccessKeyID,
+ SecretAccessKey: cmp.Or(updated.SecretAccessKey, configuration.SecretAccessKey),
+ Prefix: updated.Prefix,
+ UseSSL: updated.UseSSL,
+ ForcePathStyle: updated.ForcePathStyle,
+ })
}
func (s *S3DestinationService) TestS3DestinationConfiguration(ctx context.Context, input backup.CreateS3Destination) error {
if err := normalization.Normalize(&input); err != nil {
return err
}
- return s3.TestConnection(ctx, destinationConfigurationInternal("", input))
+ return s3.TestConnection(ctx, destinationConfiguration("", input))
}
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
index e4b6dc405..acb492c31 100644
--- a/backend/internal/s3/service_test.go
+++ b/backend/internal/s3/service_test.go
@@ -1,6 +1,7 @@
package s3
import (
+ "context"
"io"
"net/http"
"net/http/httptest"
@@ -23,7 +24,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
)
-func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService, *gorm.DB) {
+func setupS3DestinationServiceTest(t *testing.T) (*S3DestinationService, *gorm.DB) {
t.Helper()
gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
@@ -40,11 +41,11 @@ func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService,
EncryptionKey: "test-encryption-key-for-s3-destination-32bytes",
Environment: "test",
})
- return NewS3DestinationService(&database.DB{DB: gormDB}, nil), gormDB
+ return NewS3DestinationService(&database.DB{DB: gormDB}, func(context.Context, string) error { return nil }), gormDB
}
func TestS3DestinationService_CRUD(t *testing.T) {
- service, gormDB := setupS3DestinationServiceTestInternal(t)
+ service, gormDB := setupS3DestinationServiceTest(t)
ctx := t.Context()
created, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{
@@ -99,7 +100,7 @@ func TestS3DestinationService_CRUD(t *testing.T) {
}
func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {
- service, gormDB := setupS3DestinationServiceTestInternal(t)
+ service, gormDB := setupS3DestinationServiceTest(t)
ctx := t.Context()
destination, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{
Name: "In use",
@@ -117,7 +118,7 @@ func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {
}
func TestS3DestinationService_SyncS3Destinations(t *testing.T) {
- service, gormDB := setupS3DestinationServiceTestInternal(t)
+ service, gormDB := setupS3DestinationServiceTest(t)
ctx := t.Context()
legacy, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{
@@ -213,7 +214,7 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {
}))
defer server.Close()
- service, _ := setupS3DestinationServiceTestInternal(t)
+ service, _ := setupS3DestinationServiceTest(t)
require.NoError(t, service.TestS3DestinationConfiguration(t.Context(), backup.CreateS3Destination{
Name: "Unsaved destination",
Endpoint: server.URL,
@@ -344,7 +345,7 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {
}
func TestDestinationsByID(t *testing.T) {
- service, db := setupS3DestinationServiceTestInternal(t)
+ service, db := setupS3DestinationServiceTest(t)
require.Empty(t, service.DestinationsByID(t.Context()))
destination := S3Destination{Name: "Offsite", Bucket: "backups", SecretAccessKey: "encrypted"}
require.NoError(t, db.Create(&destination).Error)
@@ -354,8 +355,7 @@ func TestDestinationsByID(t *testing.T) {
require.Equal(t, "backups", indexed[destination.ID].Bucket)
require.True(t, indexed[destination.ID].SecretConfigured)
require.Empty(t, indexed["missing"].Name)
- // Listing failures and a missing service leave records undecorated rather than failing.
+ // Listing failures leave records undecorated rather than failing.
require.NoError(t, db.Migrator().DropTable(&S3Destination{}))
require.Nil(t, service.DestinationsByID(t.Context()))
- require.Nil(t, (*S3DestinationService)(nil).DestinationsByID(t.Context()))
}
diff --git a/backend/internal/system/children/backup/children/snapshots/service.go b/backend/internal/system/children/backup/children/snapshots/service.go
index e93ffe36a..697caac22 100644
--- a/backend/internal/system/children/backup/children/snapshots/service.go
+++ b/backend/internal/system/children/backup/children/snapshots/service.go
@@ -192,8 +192,11 @@ func (session systemBackupSnapshotSession) inspectProjectSnapshot(ctx context.Co
if err != nil {
return systemBackupSnapshot{}, fmt.Errorf("inspect %s project files: %w", location.name, err)
}
- snapshot.entries = projectEntriesFromSnapshot(listed, snapshot.layout, "", true)
- return snapshot, nil
+ return systemBackupSnapshot{
+ systemBackupSnapshotLocation: snapshot.systemBackupSnapshotLocation,
+ layout: snapshot.layout,
+ entries: projectEntriesFromSnapshot(listed, snapshot.layout, "", true),
+ }, nil
}
// inspectProjectManifest opens a snapshot for project browsing;
@@ -319,12 +322,12 @@ func (session systemBackupSnapshotSession) openSafetySnapshot(
systemBackupSafetySnapshot,
error,
) {
- location := source.systemBackupSnapshotLocation
- location.name = "safety"
- if source.destination == backuptypes.SystemBackupDestinationS3 {
- location.snapshotID = safetyRun.RemoteSnapshotID
- } else {
- location.snapshotID = safetyRun.LocalSnapshotID
+ location := systemBackupSnapshotLocation{ //exhaustruct:enforce
+ name: "safety",
+ destination: source.destination,
+ s3DestinationID: source.s3DestinationID,
+ repository: source.repository,
+ snapshotID: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, safetyRun.RemoteSnapshotID, safetyRun.LocalSnapshotID),
}
if location.snapshotID == "" {
return systemBackupSafetySnapshot{}, errors.New("pre-restore system backup has no snapshot in the selected repository")
@@ -337,7 +340,11 @@ func (session systemBackupSnapshotSession) openSafetySnapshot(
for _, entry := range snapshot.entries {
paths[entry.Path] = struct{}{}
}
- return systemBackupSafetySnapshot{systemBackupSnapshotLocation: location, layout: snapshot.layout, paths: paths}, nil
+ return systemBackupSafetySnapshot{
+ systemBackupSnapshotLocation: location,
+ layout: snapshot.layout,
+ paths: paths,
+ }, nil
}
func (session systemBackupSnapshotSession) restoreEntry(
@@ -437,9 +444,7 @@ func (session systemBackupSnapshotSession) restoreSelected(
restored := make([]backuptypes.BackupFileEntry, 0, len(selected))
for _, selectedEntry := range selected {
if err := session.restoreEntry(ctx, snapshots, selectedEntry, destination); err != nil {
- affected := slices.Clone(restored)
- affected = append(affected, selectedEntry)
- rollbackErr := session.rollback(context.WithoutCancel(ctx), safety, affected, destination)
+ rollbackErr := session.rollback(context.WithoutCancel(ctx), safety, append(slices.Clone(restored), selectedEntry), destination)
restoreErr := fmt.Errorf("failed to restore project path %s from system backup: %w", selectedEntry.Path, err)
if rollbackErr != nil {
return errors.Join(restoreErr, fmt.Errorf("failed to roll back project files from pre-restore system backup: %w", rollbackErr))
@@ -492,12 +497,11 @@ func (s *Service) RestoreFiles(
// The safety backup mirrors the restore source so the rollback snapshot
// lives in the repository the restore is already reading from.
source := snapshots[0]
- safetyRequest := backuptypes.CreateSystemBackupRequest{Destination: backuptypes.SystemBackupDestinationLocal, RecoveryKey: key}
- if source.destination == backuptypes.SystemBackupDestinationS3 {
- safetyRequest.Destination = backuptypes.SystemBackupDestinationS3
- safetyRequest.S3DestinationID = source.s3DestinationID
- }
- safetyRun, err := createSafety(ctx, safetyRequest)
+ safetyRun, err := createSafety(ctx, backuptypes.CreateSystemBackupRequest{
+ Destination: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationLocal),
+ S3DestinationID: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, source.s3DestinationID, ""),
+ RecoveryKey: key,
+ })
if err != nil {
return fmt.Errorf("failed to create pre-restore system backup: %w", err)
}
@@ -552,7 +556,12 @@ func (s *Service) PlanRollback(
return nil, err
}
session := s.snapshotSession(dockerClient, key)
- safetyLocation := systemBackupSnapshotLocation{name: "safety", destination: backuptypes.SystemBackupDestinationLocal, repository: localRepository, snapshotID: localSnapshotID}
+ safetyLocation := systemBackupSnapshotLocation{
+ name: "safety",
+ destination: backuptypes.SystemBackupDestinationLocal,
+ repository: localRepository,
+ snapshotID: localSnapshotID,
+ }
safetyLayout, err := session.readSnapshotLayout(ctx, safetyLocation)
if err != nil {
return nil, fmt.Errorf("open pre-restore system backup: %w", err)
@@ -631,38 +640,24 @@ func sourceMounts(mounts []container.MountPoint, inContainer bool, containerPath
if primary == nil {
return nil, fmt.Errorf("%s must be mounted into the Arcane container from a bind or named volume for system backups", containerPath)
}
- result := append([]mount.Mount{*primary}, docker.NestedMounts(mounts, containerPath, target)...)
- for index := range result {
- result[index].ReadOnly = true
+ result := make([]mount.Mount, 0, 1+len(docker.NestedMounts(mounts, containerPath, target)))
+ for _, resolved := range append([]mount.Mount{*primary}, docker.NestedMounts(mounts, containerPath, target)...) {
+ result = append(result, mount.Mount{ //exhaustruct:enforce
+ Type: resolved.Type,
+ Source: resolved.Source,
+ Target: resolved.Target,
+ ReadOnly: true,
+ Consistency: resolved.Consistency,
+ BindOptions: resolved.BindOptions,
+ VolumeOptions: resolved.VolumeOptions,
+ ImageOptions: resolved.ImageOptions,
+ TmpfsOptions: resolved.TmpfsOptions,
+ ClusterOptions: resolved.ClusterOptions,
+ })
}
return result, nil
}
-// bindVolumeSubpaths rewrites volume subpath mounts as binds of the
-// subpath on daemons before API 1.45, which ignore subpaths and mount the
-// volume root instead.
-func bindVolumeSubpaths(ctx context.Context, dockerClient *client.Client, mounts []mount.Mount) error {
- if compat.IsDockerAPIVersionAtLeast(compat.DetectDockerAPIVersion(ctx, dockerClient), "1.45") {
- return nil
- }
- for index, resolved := range mounts {
- if resolved.Type != mount.TypeVolume || resolved.VolumeOptions == nil || resolved.VolumeOptions.Subpath == "" {
- continue
- }
- inspect, err := dockerClient.VolumeInspect(ctx, resolved.Source, client.VolumeInspectOptions{})
- if err != nil {
- return fmt.Errorf("inspect volume %s for system backup: %w", resolved.Source, err)
- }
- mounts[index] = mount.Mount{
- Type: mount.TypeBind,
- Source: filepath.Join(inspect.Volume.Mountpoint, resolved.VolumeOptions.Subpath),
- Target: resolved.Target,
- ReadOnly: resolved.ReadOnly,
- }
- }
- return nil
-}
-
// nestedDataPath returns the data directory relative to the projects
// directory when the projects mount contains it on the host: the same bind
// path prefix or the same named volume and subpath. Mounting one host
@@ -678,14 +673,11 @@ func nestedDataPath(mounts []container.MountPoint, dataDirectory, projectsDirect
if resolved == nil {
return "", false, nil
}
- source := path.Clean(filepath.ToSlash(resolved.Source))
- if resolved.Type == mount.TypeVolume {
- source = "volume:" + resolved.Source
- if resolved.VolumeOptions != nil && resolved.VolumeOptions.Subpath != "" {
- source = path.Join(source, resolved.VolumeOptions.Subpath)
- }
- }
- sources = append(sources, source)
+ sources = append(sources, kit.Ternary(
+ resolved.Type == mount.TypeVolume,
+ path.Join("volume:"+resolved.Source, kit.FromPtr(resolved.VolumeOptions).Subpath),
+ path.Clean(filepath.ToSlash(resolved.Source)),
+ ))
}
projectsSource, dataSource := sources[0], sources[1]
if !kit.FilePathMatches(dataSource, projectsSource) {
@@ -707,42 +699,54 @@ func (s *Service) backupSourceLayout(ctx context.Context, dockerClient *client.C
if err != nil {
return backupSourceLayout{}, err
}
- layout := backupSourceLayout{
- dataDirectory: filepath.Dir(databaseFile),
- projectsDirectory: s.projectsDirectory(ctx),
- databaseName: filepath.Base(databaseFile),
- sources: []string{snapshotDataPath},
+ dataDirectory, projectsDirectory, databaseName := filepath.Dir(databaseFile), s.projectsDirectory(ctx), filepath.Base(databaseFile)
+ excludes := []string{".arcane-snapshot-*", RecoveryRequestName}
+ for _, name := range []string{databaseName, francis.StorePath(databaseName)} {
+ for _, sidecar := range sqliteSidecars {
+ excludes = append(excludes, name+sidecar)
+ }
}
- projectsPath, external, err := projectsSnapshotPath(layout.dataDirectory, layout.projectsDirectory)
+ projectsPath, external, err := projectsSnapshotPath(dataDirectory, projectsDirectory)
if err != nil {
return backupSourceLayout{}, err
}
- layout.projectsPath = projectsPath
mounts, inContainer, err := currentMounts(ctx, dockerClient)
if err != nil {
return backupSourceLayout{}, err
}
- layout.mounts, err = sourceMounts(mounts, inContainer, layout.dataDirectory, snapshotDataPath)
+ dataMounts, err := sourceMounts(mounts, inContainer, dataDirectory, snapshotDataPath)
if err != nil {
return backupSourceLayout{}, fmt.Errorf("resolve Arcane data source: %w", err)
}
if !external {
- return layout, nil
- }
- projectMounts, err := sourceMounts(mounts, inContainer, layout.projectsDirectory, snapshotProjectsPath)
+ return backupSourceLayout{
+ dataDirectory: dataDirectory,
+ projectsDirectory: projectsDirectory,
+ databaseName: databaseName,
+ projectsPath: projectsPath,
+ mounts: dataMounts,
+ sources: []string{snapshotDataPath},
+ excludes: excludes,
+ }, nil
+ }
+ projectMounts, err := sourceMounts(mounts, inContainer, projectsDirectory, snapshotProjectsPath)
if err != nil {
return backupSourceLayout{}, fmt.Errorf("resolve projects source: %w", err)
}
- layout.mounts = append(layout.mounts, projectMounts...)
- layout.sources = append(layout.sources, snapshotProjectsPath)
- relative, nested, err := nestedDataPath(mounts, layout.dataDirectory, layout.projectsDirectory)
+ relative, nested, err := nestedDataPath(mounts, dataDirectory, projectsDirectory)
if err != nil {
return backupSourceLayout{}, err
}
- if nested {
- layout.nestedDataPath = path.Join(snapshotProjectsPath, relative)
- }
- return layout, nil
+ return backupSourceLayout{
+ dataDirectory: dataDirectory,
+ projectsDirectory: projectsDirectory,
+ databaseName: databaseName,
+ projectsPath: projectsPath,
+ nestedDataPath: kit.Ternary(nested, path.Join(snapshotProjectsPath, relative), ""),
+ mounts: append(dataMounts, projectMounts...),
+ sources: []string{snapshotDataPath, snapshotProjectsPath},
+ excludes: excludes,
+ }, nil
}
var (
@@ -840,16 +844,18 @@ func snapshotLayoutFromManifest(manifest recovery.Manifest, manifestRoot string)
if err != nil {
return snapshotLayout{}, err
}
- layout := snapshotLayout{dataPath: manifestRoot, databaseName: path.Base(databasePath)}
relative, err := projectsRelativePathFromManifest(manifest)
if errors.Is(err, errProjectsOutsideData) {
- return layout, nil
+ return snapshotLayout{dataPath: manifestRoot, databaseName: path.Base(databasePath)}, nil
}
if err != nil {
return snapshotLayout{}, fmt.Errorf("resolve projects directory: %w", err)
}
- layout.projectsPath = path.Join(manifestRoot, relative)
- return layout, nil
+ return snapshotLayout{
+ dataPath: manifestRoot,
+ projectsPath: path.Join(manifestRoot, relative),
+ databaseName: path.Base(databasePath),
+ }, nil
case recovery.ManifestFormatVersion:
dataPath, err := kit.NormalizeRelativePath(strings.TrimPrefix(manifest.DataPath, "/"))
if err != nil {
@@ -955,9 +961,10 @@ func restoreTarget(mounts []container.MountPoint, containerPath, target, exclude
candidates := slices.DeleteFunc(slices.Clone(mounts), func(m container.MountPoint) bool {
return exclude != "" && kit.FilePathMatches(m.Destination, exclude)
})
- result := recovery.RestoreTarget{Mounts: []mount.Mount{*enclosing}, Path: destination}
- result.Mounts = append(result.Mounts, docker.NestedMounts(candidates, containerPath, destination)...)
- return result, nil
+ return recovery.RestoreTarget{
+ Mounts: append([]mount.Mount{*enclosing}, docker.NestedMounts(candidates, containerPath, destination)...),
+ Path: destination,
+ }, nil
}
// projectsRestoreDestination is the current projects directory as a
@@ -975,11 +982,17 @@ func (s *Service) projectsRestoreDestination(ctx context.Context, dockerClient *
return projectsRestoreDestination{}, err
}
if !inContainer {
- hostTarget := recovery.RestoreTarget{
- Mounts: []mount.Mount{{Type: mount.TypeBind, Source: directory, Target: selectiveProjectsRestoreTarget}},
- Path: selectiveProjectsRestoreTarget,
- }
- return projectsRestoreDestination{directory: directory, target: hostTarget}, nil
+ return projectsRestoreDestination{
+ directory: directory,
+ target: recovery.RestoreTarget{
+ Mounts: []mount.Mount{{
+ Type: mount.TypeBind,
+ Source: directory,
+ Target: selectiveProjectsRestoreTarget,
+ }},
+ Path: selectiveProjectsRestoreTarget,
+ },
+ }, nil
}
target, err := restoreTarget(mounts, directory, selectiveProjectsRestoreTarget, "")
if err != nil {
@@ -989,12 +1002,15 @@ func (s *Service) projectsRestoreDestination(ctx context.Context, dockerClient *
if err != nil {
return projectsRestoreDestination{}, err
}
- dataDirectory := filepath.Dir(databaseFile)
- dataPath, _, err := nestedDataPath(mounts, dataDirectory, directory)
+ dataPath, _, err := nestedDataPath(mounts, filepath.Dir(databaseFile), directory)
if err != nil {
return projectsRestoreDestination{}, err
}
- return projectsRestoreDestination{directory: directory, dataPath: dataPath, target: target}, nil
+ return projectsRestoreDestination{
+ directory: directory,
+ dataPath: dataPath,
+ target: target,
+ }, nil
}
// restoreStages plans the Rustic restores that put a snapshot's data
@@ -1032,8 +1048,12 @@ func restoreStages(
projectsDirectory, dataDirectory,
)
}
- stages := make([]recovery.RestoreStage, 0, 2)
- stages = append(stages, recovery.RestoreStage{Repository: repository, SnapshotID: snapshotID, SourcePath: layout.dataPath, Target: dataTarget})
+ stages := []recovery.RestoreStage{{
+ Repository: repository,
+ SnapshotID: snapshotID,
+ SourcePath: layout.dataPath,
+ Target: dataTarget,
+ }}
if !separate {
return stages, nil
}
@@ -1041,7 +1061,12 @@ func restoreStages(
if err != nil {
return nil, err
}
- return append(stages, recovery.RestoreStage{Repository: repository, SnapshotID: snapshotID, SourcePath: layout.projectsPath, Target: projectsTarget}), nil
+ return append(stages, recovery.RestoreStage{
+ Repository: repository,
+ SnapshotID: snapshotID,
+ SourcePath: layout.projectsPath,
+ Target: projectsTarget,
+ }), nil
}
// Create archives a consistent system snapshot without blocking actor leases during the upload.
@@ -1091,12 +1116,6 @@ func (s *Service) Create(
return backup.Snapshot{}, fmt.Errorf("failed to write recovery manifest: %w", err)
}
defer func() { _ = os.Remove(manifestPath) }()
- layout.excludes = []string{".arcane-snapshot-*", RecoveryRequestName}
- for name := range databases {
- for _, sidecar := range sqliteSidecars {
- layout.excludes = append(layout.excludes, name+sidecar)
- }
- }
files, err := snapshotSourceFiles(ctx, layout)
if err != nil {
return backup.Snapshot{}, err
@@ -1106,6 +1125,7 @@ func (s *Service) Create(
return backup.Snapshot{}, err
}
// Mount VACUUM INTO copies over the live databases so the snapshot is consistent.
+ snapshotMounts := slices.Clone(layout.mounts)
for name, db := range databases {
stagedDatabase := filepath.Join(stage, name)
if stageErr := stageDatabase(ctx, db, filepath.Join(layout.dataDirectory, name), stagedDatabase); stageErr != nil {
@@ -1115,19 +1135,40 @@ func (s *Service) Create(
if mountErr != nil {
return backup.Snapshot{}, mountErr
}
- layout.mounts = append(layout.mounts, stagedMounts...)
- }
- if bindErr := bindVolumeSubpaths(ctx, dockerClient, layout.mounts); bindErr != nil {
- return backup.Snapshot{}, bindErr
+ snapshotMounts = append(snapshotMounts, stagedMounts...)
+ }
+ // Rewrite volume subpath mounts as binds of the
+ // subpath on daemons before API 1.45, which ignore subpaths and mount the
+ // volume root instead.
+ if !compat.IsDockerAPIVersionAtLeast(compat.DetectDockerAPIVersion(ctx, dockerClient), "1.45") {
+ for index, resolved := range snapshotMounts {
+ if resolved.Type != mount.TypeVolume || kit.FromPtr(resolved.VolumeOptions).Subpath == "" {
+ continue
+ }
+ inspect, inspectErr := dockerClient.VolumeInspect(ctx, resolved.Source, client.VolumeInspectOptions{})
+ if inspectErr != nil {
+ return backup.Snapshot{}, fmt.Errorf("inspect volume %s for system backup: %w", resolved.Source, inspectErr)
+ }
+ snapshotMounts[index] = mount.Mount{
+ Type: mount.TypeBind,
+ Source: filepath.Join(inspect.Volume.Mountpoint, resolved.VolumeOptions.Subpath),
+ Target: resolved.Target,
+ ReadOnly: resolved.ReadOnly,
+ }
+ }
}
- input := backup.CreateSnapshotInput{Mounts: layout.mounts, Sources: layout.sources}
+ globs := make([]string, 0, len(layout.excludes)+1)
for _, excluded := range layout.excludes {
- input.Globs = append(input.Globs, "!"+snapshotDataPath+"/"+excluded)
+ globs = append(globs, "!"+snapshotDataPath+"/"+excluded)
}
if layout.nestedDataPath != "" {
- input.Globs = append(input.Globs, "!"+layout.nestedDataPath)
+ globs = append(globs, "!"+layout.nestedDataPath)
}
- snapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", input)
+ snapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", backup.CreateSnapshotInput{
+ Mounts: snapshotMounts,
+ Sources: layout.sources,
+ Globs: globs,
+ })
if err != nil {
return backup.Snapshot{}, err
}
@@ -1150,11 +1191,11 @@ func (s *Service) Create(
}
}
if err == nil {
- var confirmed backup.Snapshot
- confirmed, err = s.engine.ConfirmRunSnapshot(ctx, dockerClient, repository, recoveryKey, backupID, snapshot.ID)
- if err == nil {
+ confirmed, confirmErr := s.engine.ConfirmRunSnapshot(ctx, dockerClient, repository, recoveryKey, backupID, snapshot.ID)
+ if confirmErr == nil {
return confirmed, nil
}
+ err = confirmErr
}
// Tagging changes the ID; only remove the original, unconfirmed snapshot.
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
@@ -1171,9 +1212,6 @@ func (s *Service) Create(
// project folders hold live container data.
func snapshotSourceFiles(ctx context.Context, layout backupSourceLayout) (map[string]os.FileInfo, error) {
projectsInfo, statErr := os.Stat(layout.projectsDirectory)
- if statErr != nil {
- projectsInfo = nil
- }
actorDatabase := francis.StorePath(layout.databaseName)
files := make(map[string]os.FileInfo)
visit := func(filePath string, entry fs.DirEntry, walkErr error) error {
@@ -1207,7 +1245,7 @@ func snapshotSourceFiles(ctx context.Context, layout backupSourceLayout) (map[st
if err != nil {
return err
}
- if entry.IsDir() && relative != "." && projectsInfo != nil && os.SameFile(info, projectsInfo) {
+ if entry.IsDir() && relative != "." && statErr == nil && os.SameFile(info, projectsInfo) {
return filepath.SkipDir
}
files[filePath] = info
diff --git a/backend/internal/system/children/backup/children/volumes/handler.go b/backend/internal/system/children/backup/children/volumes/handler.go
index 243f69b7c..05d48376e 100644
--- a/backend/internal/system/children/backup/children/volumes/handler.go
+++ b/backend/internal/system/children/backup/children/volumes/handler.go
@@ -2,11 +2,12 @@ package volumes
import (
"context"
- "errors"
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/backup"
+ kit "go.getarcane.app/kit/pkg"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
@@ -66,20 +67,14 @@ func (h *Handler) ListSystemVolumeOptions(ctx context.Context, _ *struct{}) (*Sy
}
func (h *Handler) RunSystemVolumeBackups(ctx context.Context, input *RunSystemVolumeBackupsInput) (*SystemVolumeBackupRunOutput, error) {
- request := backup.RunSystemVolumeBackupsRequest{}
- if input.Body != nil {
- request = *input.Body
- }
user, err := handlerutil.RequireUser(ctx)
if err != nil {
return nil, err
}
- result, err := h.service.Start(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, request)
- if errors.Is(err, h.service.alreadyRunning) {
- return nil, huma.Error409Conflict(err.Error())
- }
+ result, err := h.service.Start(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body))
if err != nil {
- return nil, huma.Error500InternalServerError(err.Error())
+ apiErr := common.ToAPIError(err)
+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)
}
return &SystemVolumeBackupRunOutput{Body: *result}, nil
}
diff --git a/backend/internal/system/children/backup/children/volumes/service.go b/backend/internal/system/children/backup/children/volumes/service.go
index ba9047407..35e54f236 100644
--- a/backend/internal/system/children/backup/children/volumes/service.go
+++ b/backend/internal/system/children/backup/children/volumes/service.go
@@ -1,6 +1,7 @@
package volumes
import (
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -98,15 +99,20 @@ func NewService(deps Dependencies) *Service {
func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
var err error
s.flow = engine
- template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceID: new("volumes"), ResourceName: new("Volumes")}
s.backupWorkflow, err = engine.Define(flow.Definition{
Name: "system-volume-backup",
Version: 1,
Fingerprint: "9b4a5e46a984446bc08df39a71e563e031548db3af1825a2413e8c96515db6ab",
Concurrency: 1,
Timeout: 24 * time.Hour,
- Activity: template,
- Labels: map[string]string{"backup": "Backing up volumes"},
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeResourceAction,
+ EnvironmentID: "0",
+ ResourceType: new("system_backup"),
+ ResourceID: new("volumes"),
+ ResourceName: new("Volumes"),
+ },
+ Labels: map[string]string{"backup": "Backing up volumes"},
Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
var input selectionInput
if decodeErr := t.Payload(&input); decodeErr != nil {
@@ -122,15 +128,23 @@ func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
if err != nil {
return err
}
- template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Backing up volumes", "Creating scheduled system-managed volume backups"
s.policyWorkflow, err = engine.Define(flow.Definition{
Name: "system-volume-backup-policy",
Version: 1,
Fingerprint: "94a13d1bac530e6eddcb73c27990a07d69915dc947ce5b559166eb3fb85e39ef",
Concurrency: 1,
Timeout: 24 * time.Hour,
- Activity: template,
- Labels: map[string]string{"backup": "Backing up volumes"},
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeResourceAction,
+ EnvironmentID: "0",
+ ResourceType: new("system_backup"),
+ ResourceID: new("volumes"),
+ ResourceName: new("Volumes"),
+ StartedBy: &usertypes.SystemUser,
+ Step: "Backing up volumes",
+ LatestMessage: "Creating scheduled system-managed volume backups",
+ },
+ Labels: map[string]string{"backup": "Backing up volumes"},
Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
previous, _ := jobcontext.Run(ctx)
if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == systemVolumePlanTarget }) {
@@ -147,21 +161,15 @@ func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
}
func (s *Service) loadSystemVolumeBackupPolicies() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {
- collection := &backuptypes.SystemVolumeBackupPolicyCollection{Policies: []backuptypes.SystemVolumeBackupPolicy{}}
- if s.settings == nil {
- return collection, nil
- }
- raw := strings.TrimSpace(s.settings.GetSettingsConfig().SystemVolumeBackupConfig.Value)
- if raw == "" {
- return collection, nil
- }
- if err := json.Unmarshal([]byte(raw), collection); err != nil {
- return nil, fmt.Errorf("decode system-managed volume backup policies: %w", err)
- }
- if collection.Policies == nil {
- collection.Policies = []backuptypes.SystemVolumeBackupPolicy{}
+ var collection backuptypes.SystemVolumeBackupPolicyCollection
+ if raw := strings.TrimSpace(s.settings.GetSettingsConfig().SystemVolumeBackupConfig.Value); raw != "" {
+ if err := json.Unmarshal([]byte(raw), &collection); err != nil {
+ return nil, fmt.Errorf("decode system-managed volume backup policies: %w", err)
+ }
}
- return collection, nil
+ return &backuptypes.SystemVolumeBackupPolicyCollection{
+ Policies: kit.Ternary(collection.Policies == nil, []backuptypes.SystemVolumeBackupPolicy{}, collection.Policies),
+ }, nil
}
func (s *Service) systemVolumeBackupPolicy(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {
@@ -182,34 +190,57 @@ func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBacku
return nil, err
}
destinations := s.s3Destinations.DestinationsByID(ctx)
- for i := range collection.Policies {
- policy := &collection.Policies[i]
- policy.S3DestinationName = destinations[policy.S3DestinationID].Name
- if s.db == nil {
- continue
- }
+ policies := make([]backuptypes.SystemVolumeBackupPolicy, 0, len(collection.Policies))
+ for _, policy := range collection.Policies {
var lastRun volume.VolumeBackup
runErr := s.db.WithContext(ctx).
Where("policy_id LIKE ?", backuptypes.SystemVolumePolicyPrefix+policy.ID+":%").
Order("created_at DESC").First(&lastRun).Error
- if runErr == nil {
- policy.LastRun = &backuptypes.SystemBackupRun{
- ID: lastRun.ID, Size: lastRun.Size, CreatedAt: lastRun.CreatedAt, Status: string(lastRun.Status),
- Trigger: string(lastRun.Trigger), Destination: backuptypes.SystemBackupDestination(lastRun.Destination),
- LocalSnapshotID: lastRun.LocalSnapshotID, RemoteSnapshotID: lastRun.RemoteSnapshotID,
- S3DestinationID: lastRun.S3DestinationID, S3DestinationName: destinations[lastRun.S3DestinationID].Name,
- PolicyID: lastRun.PolicyID, Error: lastRun.Error,
- }
- } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {
+ if runErr != nil && !errors.Is(runErr, gorm.ErrRecordNotFound) {
return nil, fmt.Errorf("load latest system-managed volume backup: %w", runErr)
}
+ policies = append(policies, backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce
+ ID: policy.ID,
+ Enabled: policy.Enabled,
+ Schedule: policy.Schedule,
+ RetentionCount: policy.RetentionCount,
+ StopContainers: policy.StopContainers,
+ LocalEnabled: policy.LocalEnabled,
+ S3Enabled: policy.S3Enabled,
+ S3DestinationID: policy.S3DestinationID,
+ S3DestinationName: destinations[policy.S3DestinationID].Name,
+ SelectionMode: policy.SelectionMode,
+ VolumeNames: policy.VolumeNames,
+ IgnoreAnonymous: policy.IgnoreAnonymous,
+ LastRun: func() *backuptypes.SystemBackupRun {
+ if runErr != nil {
+ return policy.LastRun
+ }
+ return &backuptypes.SystemBackupRun{
+ ID: lastRun.ID,
+ Size: lastRun.Size,
+ CreatedAt: lastRun.CreatedAt,
+ Status: string(lastRun.Status),
+ Trigger: string(lastRun.Trigger),
+ Destination: backuptypes.SystemBackupDestination(lastRun.Destination),
+ LocalSnapshotID: lastRun.LocalSnapshotID,
+ RemoteSnapshotID: lastRun.RemoteSnapshotID,
+ S3DestinationID: lastRun.S3DestinationID,
+ S3DestinationName: destinations[lastRun.S3DestinationID].Name,
+ PolicyID: lastRun.PolicyID,
+ Error: lastRun.Error,
+ }
+ }(),
+ })
}
- return collection, nil
+ return &backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies}, nil
}
func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {
- mode := input.SelectionMode
- if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {
+ if !slices.Contains(
+ []backuptypes.SystemVolumeSelectionMode{backuptypes.SystemVolumeSelectionAll, backuptypes.SystemVolumeSelectionAllowlist, backuptypes.SystemVolumeSelectionBlocklist},
+ input.SelectionMode,
+ ) {
return backuptypes.SystemVolumeBackupPolicy{}, errors.New("selectionMode must be all, allowlist, or blocklist")
}
update, err := backup.ValidatePolicyUpdate(ctx, "system-managed volume", input.UpdateBackupPolicy, s.s3Destinations)
@@ -218,26 +249,27 @@ func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input b
}
names := kit.Unique(kit.TrimNonEmpty(input.VolumeNames))
slices.Sort(names)
- if mode == backuptypes.SystemVolumeSelectionAll || names == nil {
- names = []string{}
- }
return backuptypes.SystemVolumeBackupPolicy{
- ID: input.ID, Enabled: update.Enabled, Schedule: update.Schedule, RetentionCount: update.RetentionCount,
- StopContainers: update.StopContainers, LocalEnabled: update.LocalEnabled, S3Enabled: update.S3Enabled,
- S3DestinationID: update.S3DestinationID, SelectionMode: mode,
- VolumeNames: names, IgnoreAnonymous: input.IgnoreAnonymous,
+ ID: input.ID,
+ Enabled: update.Enabled,
+ Schedule: update.Schedule,
+ RetentionCount: update.RetentionCount,
+ StopContainers: update.StopContainers,
+ LocalEnabled: update.LocalEnabled,
+ S3Enabled: update.S3Enabled,
+ S3DestinationID: update.S3DestinationID,
+ SelectionMode: input.SelectionMode,
+ VolumeNames: kit.Ternary(input.SelectionMode == backuptypes.SystemVolumeSelectionAll || names == nil, []string{}, names),
+ IgnoreAnonymous: input.IgnoreAnonymous,
}, nil
}
func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.UpdateSystemVolumeBackupPolicy) (*backuptypes.SystemVolumeBackupPolicyCollection, error) {
- if s.settings == nil {
- return nil, errors.New("settings service is unavailable")
- }
existing, err := s.loadSystemVolumeBackupPolicies()
if err != nil {
return nil, err
}
- reconcile := backup.PolicyReconciliation[backuptypes.SystemVolumeBackupPolicy, backuptypes.UpdateSystemVolumeBackupPolicy]{
+ if runErr := (backup.PolicyReconciliation[backuptypes.SystemVolumeBackupPolicy, backuptypes.UpdateSystemVolumeBackupPolicy]{
Domain: "system-managed volume",
Existing: existing.Policies,
ID: func(policy *backuptypes.SystemVolumeBackupPolicy) string { return policy.ID },
@@ -250,8 +282,21 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
if normalizeErr != nil {
return normalizeErr
}
- normalized.ID = policy.ID
- *policy = normalized
+ *policy = backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce
+ ID: policy.ID,
+ Enabled: normalized.Enabled,
+ Schedule: normalized.Schedule,
+ RetentionCount: normalized.RetentionCount,
+ StopContainers: normalized.StopContainers,
+ LocalEnabled: normalized.LocalEnabled,
+ S3Enabled: normalized.S3Enabled,
+ S3DestinationID: normalized.S3DestinationID,
+ S3DestinationName: normalized.S3DestinationName,
+ SelectionMode: normalized.SelectionMode,
+ VolumeNames: normalized.VolumeNames,
+ IgnoreAnonymous: normalized.IgnoreAnonymous,
+ LastRun: normalized.LastRun,
+ }
return nil
},
Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {
@@ -265,8 +310,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)
},
Reschedule: s.rescheduleSystemVolumeBackup,
- }
- if runErr := reconcile.Run(ctx, updates); runErr != nil {
+ }).Run(ctx, updates); runErr != nil {
return nil, runErr
}
return s.GetConfig(ctx)
@@ -274,9 +318,6 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
// ListOptions returns live choices plus configured names that are currently unavailable.
func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBackupOption, error) {
- if s.volumes == nil {
- return nil, errors.New("volume service is unavailable")
- }
options, err := s.volumes.ListBackupVolumeOptions(ctx)
if err != nil {
return nil, err
@@ -305,44 +346,53 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBa
// prepareSystemVolumeBackups resolves the requested policy, takes the system admission, and selects the live volumes.
func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (selectionInput, *runs.Lease, error) {
- if s.volumes == nil {
- return selectionInput{}, nil, errors.New("volume service is unavailable")
- }
- var input selectionInput
- switch {
- case request.PolicyID != "" && request.Custom != nil:
+ if request.PolicyID != "" && request.Custom != nil {
return selectionInput{}, nil, errors.New("select a saved policy or custom configuration, not both")
- case request.PolicyID != "":
- policy, err := s.systemVolumeBackupPolicy(request.PolicyID)
- if err != nil {
- return selectionInput{}, nil, err
- }
- if policy == nil {
- return selectionInput{}, nil, errors.New("system-managed volume backup policy not found")
- }
- input.Policy = *policy
- default:
- custom := backuptypes.UpdateSystemVolumeBackupPolicy{
- Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,
- SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,
- }
- if run := request.Custom; run != nil {
- localEnabled := run.Destination == backuptypes.SystemBackupDestinationLocal || run.Destination == backuptypes.SystemBackupDestinationLocalS3
- s3Enabled := run.Destination == backuptypes.SystemBackupDestinationS3 || run.Destination == backuptypes.SystemBackupDestinationLocalS3
- if !localEnabled && !s3Enabled {
- return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")
- }
- custom = backuptypes.UpdateSystemVolumeBackupPolicy{
- Enabled: true, Schedule: defaultSystemVolumeSchedule, StopContainers: run.StopContainers,
- LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: run.S3DestinationID,
- SelectionMode: run.SelectionMode, VolumeNames: run.VolumeNames, IgnoreAnonymous: run.IgnoreAnonymous,
+ }
+ if request.Custom != nil && !slices.Contains(
+ []backuptypes.SystemBackupDestination{backuptypes.SystemBackupDestinationLocal, backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationLocalS3},
+ request.Custom.Destination,
+ ) {
+ return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")
+ }
+ policy, err := func() (*backuptypes.SystemVolumeBackupPolicy, error) {
+ if request.PolicyID != "" {
+ stored, loadErr := s.systemVolumeBackupPolicy(request.PolicyID)
+ if loadErr == nil && stored == nil {
+ return nil, errors.New("system-managed volume backup policy not found")
}
- }
- policy, err := s.normalizeSystemVolumePolicyUpdate(ctx, custom)
- if err != nil {
- return selectionInput{}, nil, err
- }
- input.Policy, input.ManualPolicy = policy, true
+ return stored, loadErr
+ }
+ if request.Custom == nil {
+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, backuptypes.UpdateSystemVolumeBackupPolicy{
+ UpdateBackupPolicy: backuptypes.UpdateBackupPolicy{
+ Enabled: true,
+ Schedule: defaultSystemVolumeSchedule,
+ LocalEnabled: true,
+ },
+ SelectionMode: backuptypes.SystemVolumeSelectionAll,
+ VolumeNames: []string{},
+ IgnoreAnonymous: true,
+ })
+ return &normalized, normalizeErr
+ }
+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, backuptypes.UpdateSystemVolumeBackupPolicy{
+ UpdateBackupPolicy: backuptypes.UpdateBackupPolicy{
+ Enabled: true,
+ Schedule: defaultSystemVolumeSchedule,
+ StopContainers: request.Custom.StopContainers,
+ LocalEnabled: request.Custom.Destination == backuptypes.SystemBackupDestinationLocal || request.Custom.Destination == backuptypes.SystemBackupDestinationLocalS3,
+ S3Enabled: request.Custom.Destination == backuptypes.SystemBackupDestinationS3 || request.Custom.Destination == backuptypes.SystemBackupDestinationLocalS3,
+ S3DestinationID: request.Custom.S3DestinationID,
+ },
+ SelectionMode: request.Custom.SelectionMode,
+ VolumeNames: request.Custom.VolumeNames,
+ IgnoreAnonymous: request.Custom.IgnoreAnonymous,
+ })
+ return &normalized, normalizeErr
+ }()
+ if err != nil {
+ return selectionInput{}, nil, err
}
lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)
if err != nil {
@@ -356,14 +406,17 @@ func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backup
lease.Release(ctx)
return selectionInput{}, nil, err
}
- input.Candidates = selectSystemVolumeBackupCandidates(input.Policy, options)
- return input, lease, nil
+ return selectionInput{
+ Policy: *policy,
+ ManualPolicy: request.PolicyID == "",
+ Candidates: selectSystemVolumeBackupCandidates(*policy, options),
+ }, lease, nil
}
func (s *Service) executeSystemVolumeBackups(
ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string,
) (result *backuptypes.SystemVolumeBackupRunResult, err error) {
- result = &backuptypes.SystemVolumeBackupRunResult{Matched: len(input.Candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0)}
+ succeeded, failed, skipped, failures := 0, 0, 0, make([]backuptypes.SystemVolumeBackupFailure, 0)
reportProgress := func(ctx context.Context) {
if activityID == "" {
return
@@ -373,18 +426,36 @@ func (s *Service) executeSystemVolumeBackups(
names[i] = candidate.Name
}
progress := 100
- if result.Matched > 0 {
- progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched
- }
- _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{
- "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names,
- "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,
- }})
+ if len(input.Candidates) > 0 {
+ progress = 100 * (succeeded + failed + skipped) / len(input.Candidates)
+ }
+ _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{
+ Progress: &progress,
+ Metadata: database.JSON{
+ "action": "run_system_volume_backups",
+ "policyId": input.Policy.ID,
+ "volumeNames": names,
+ "matched": len(input.Candidates),
+ "succeeded": succeeded,
+ "failed": failed,
+ "skipped": skipped,
+ "failures": failures,
+ },
+ })
if updateErr != nil {
slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", input.Policy.ID, "error", updateErr)
}
}
defer reportProgress(context.WithoutCancel(ctx))
+ defer func() {
+ result = &backuptypes.SystemVolumeBackupRunResult{
+ Matched: len(input.Candidates),
+ Succeeded: succeeded,
+ Failed: failed,
+ Skipped: skipped,
+ Failures: failures,
+ }
+ }()
defer utils.RecoverToError(&err, "system-managed volume backup")
policy := backuptypes.UpdateBackupPolicy{
@@ -398,23 +469,23 @@ func (s *Service) executeSystemVolumeBackups(
return target.ID == candidate.Name && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)
})
if done >= 0 && previous.Outcome.Targets[done].Status == scheduler.Succeeded {
- result.Succeeded++
+ succeeded++
continue
}
if done >= 0 {
- result.Skipped++
+ skipped++
continue
}
if cancellationErr := ctx.Err(); cancellationErr != nil {
- return result, cancellationErr
+ return nil, cancellationErr
}
reportProgress(ctx)
overridden, backupErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)
if backupErr == nil && overridden {
if skipProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Skipped}); skipProgressErr != nil {
- return result, skipProgressErr
+ return nil, skipProgressErr
}
- result.Skipped++
+ skipped++
continue
}
if backupErr == nil {
@@ -422,24 +493,27 @@ func (s *Service) executeSystemVolumeBackups(
_, backupErr = s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)
}
if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {
- result.Skipped++
+ skipped++
continue
}
if backupErr != nil && ctx.Err() != nil {
// Shutdown interrupted this volume; hand the task back instead of counting it failed.
- return result, ctx.Err()
+ return nil, ctx.Err()
}
if backupErr != nil {
- result.Failed++
- result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})
+ failed++
+ failures = append(failures, backuptypes.SystemVolumeBackupFailure{
+ VolumeName: candidate.Name,
+ Error: backupErr.Error(),
+ })
continue
}
if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Succeeded}); progressErr != nil {
- return result, progressErr
+ return nil, progressErr
}
- result.Succeeded++
+ succeeded++
}
- return result, nil
+ return nil, nil
}
func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID, activityID string) (scheduler.Outcome, error) {
@@ -450,11 +524,13 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,
if policy == nil || !policy.Enabled {
return scheduler.Outcome{Status: scheduler.Skipped}, nil
}
- var remoteErr error
- if policy.S3Enabled {
+ remoteErr := func() error {
+ if !policy.S3Enabled {
+ return nil
+ }
root := path.Join(backup.VolumeRoot, s.settings.GetSettingsConfig().InstanceID.Value)
- remoteErr = backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)
- }
+ return backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)
+ }()
remoteDisabled, err := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {
collection, loadErr := s.loadSystemVolumeBackupPolicies()
if loadErr != nil {
@@ -467,8 +543,22 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,
if index < 0 {
return false, nil
}
- current := &collection.Policies[index]
- *kit.Ternary(field == "enabled", &current.Enabled, &current.S3Enabled) = false
+ current := collection.Policies[index]
+ collection.Policies[index] = backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce
+ ID: current.ID,
+ Enabled: current.Enabled && field != "enabled",
+ Schedule: current.Schedule,
+ RetentionCount: current.RetentionCount,
+ StopContainers: current.StopContainers,
+ LocalEnabled: current.LocalEnabled,
+ S3Enabled: current.S3Enabled && field != "s3_enabled",
+ S3DestinationID: current.S3DestinationID,
+ S3DestinationName: current.S3DestinationName,
+ SelectionMode: current.SelectionMode,
+ VolumeNames: current.VolumeNames,
+ IgnoreAnonymous: current.IgnoreAnonymous,
+ LastRun: current.LastRun,
+ }
encoded, encodeErr := json.Marshal(collection)
if encodeErr != nil {
return false, fmt.Errorf("encode policies: %w", encodeErr)
@@ -484,22 +574,30 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,
if remoteDisabled && !policy.LocalEnabled {
return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
}
- input, lease, err := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})
+ result, err := func() (*backuptypes.SystemVolumeBackupRunResult, error) {
+ input, lease, prepareErr := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})
+ if prepareErr != nil {
+ return nil, prepareErr
+ }
+ defer lease.Release(ctx)
+ frozen, marshalErr := json.Marshal(input)
+ if marshalErr != nil {
+ return nil, marshalErr
+ }
+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: "backup_plan",
+ ID: systemVolumePlanTarget,
+ Status: scheduler.Succeeded,
+ RecoveryData: frozen,
+ }); progressErr != nil {
+ return nil, progressErr
+ }
+ return s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)
+ }()
if errors.Is(err, s.alreadyRunning) {
slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)
return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: a system backup is running"}, nil
}
- var result *backuptypes.SystemVolumeBackupRunResult
- if err == nil {
- defer lease.Release(ctx)
- var frozen []byte
- if frozen, err = json.Marshal(input); err == nil {
- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_plan", ID: systemVolumePlanTarget, Status: scheduler.Succeeded, RecoveryData: frozen})
- }
- if err == nil {
- result, err = s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)
- }
- }
if err != nil {
slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)
return scheduler.Outcome{}, err
@@ -561,48 +659,66 @@ func (s *Service) backUpSelection(ctx context.Context, input selectionInput, tri
if outcome.Status != scheduler.Succeeded {
return nil, errors.New(outcome.Message)
}
- for i := range previous.Outcome.Targets {
- if previous.Outcome.Targets[i].ID == candidate.Name {
- previous.Outcome.Targets[i].Status = scheduler.Succeeded
+ for i, target := range previous.Outcome.Targets {
+ if target.ID == candidate.Name {
+ previous.Outcome.Targets[i] = scheduler.TargetOutcome{ //exhaustruct:enforce
+ RecoveryData: target.RecoveryData,
+ ResourceType: target.ResourceType,
+ ID: target.ID,
+ Status: scheduler.Succeeded,
+ Message: target.Message,
+ ActivityID: target.ActivityID,
+ }
}
}
}
- progressCtx := ctx
- ctx = jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(progressCtx, target) })
- return s.executeSystemVolumeBackups(ctx, input, trigger, activityID)
+ return s.executeSystemVolumeBackups(
+ jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(ctx, target) }),
+ input,
+ trigger,
+ activityID,
+ )
}
// selectionOutcome reports a run's result; failed volumes or disabled remote storage make it partial.
func selectionOutcome(result *backuptypes.SystemVolumeBackupRunResult, remoteDisabled bool) scheduler.Outcome {
- outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "System-managed volume backups completed"}
- if result.Failed > 0 {
- outcome.Status, outcome.Message = scheduler.Partial, fmt.Sprintf("%d volume backups failed", result.Failed)
- }
- if remoteDisabled {
- outcome.Status, outcome.Message = scheduler.Partial, backup.RemoteDisabledMessage
- }
+ var targets []scheduler.TargetOutcome
for _, failure := range result.Failures {
- outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})
+ targets = append(targets, scheduler.TargetOutcome{
+ ID: failure.VolumeName,
+ Status: scheduler.Failed,
+ Message: failure.Error,
+ })
+ }
+ return scheduler.Outcome{
+ Status: kit.Ternary(result.Failed > 0 || remoteDisabled, scheduler.Partial, scheduler.Succeeded),
+ Message: cmp.Or(
+ kit.Ternary(remoteDisabled, backup.RemoteDisabledMessage, ""),
+ kit.Ternary(result.Failed > 0, fmt.Sprintf("%d volume backups failed", result.Failed), ""),
+ "System-managed volume backups completed",
+ ),
+ Targets: targets,
}
- return outcome
}
func (s *Service) rescheduleSystemVolumeBackup(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {
- if policy == nil {
- return
- }
- jobID := systemVolumeBackupJobPrefix + policy.ID
if !policy.Enabled {
- s.jobs.Unregister(ctx, jobID)
+ s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policy.ID)
return
}
policyID := policy.ID
s.jobs.Add(ctx, &flow.Job{
Engine: s.flow,
Workflow: s.policyWorkflow,
- JobName: s.jobs.JobName(jobID),
+ JobName: s.jobs.JobName(systemVolumeBackupJobPrefix + policyID),
Payload: policyID,
- Activity: activitylib.StartRequest{Metadata: database.JSON{"action": "scheduled_system_volume_backups", "policyId": policyID, "schedule": policy.Schedule}},
+ Activity: activitylib.StartRequest{
+ Metadata: database.JSON{
+ "action": "scheduled_system_volume_backups",
+ "policyId": policyID,
+ "schedule": policy.Schedule,
+ },
+ },
ScheduleFn: func(context.Context) string {
current, err := s.systemVolumeBackupPolicy(policyID)
if err != nil || current == nil {
@@ -622,16 +738,21 @@ func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backu
if err := s.engine.Authorize(ctx, requester); err != nil {
return nil, err
}
- input, lease, err := s.prepareSystemVolumeBackups(ctx, request)
+ prepared, lease, err := s.prepareSystemVolumeBackups(ctx, request)
if err != nil {
return nil, err
}
- input.Requester = &requester
- names := make([]string, len(input.Candidates))
- for i, candidate := range input.Candidates {
+ names := make([]string, len(prepared.Candidates))
+ for i, candidate := range prepared.Candidates {
names[i] = candidate.Name
}
- input.Claim = uuid.NewV7().String()
+ input := selectionInput{
+ Policy: prepared.Policy,
+ ManualPolicy: prepared.ManualPolicy,
+ Candidates: prepared.Candidates,
+ Requester: &requester,
+ Claim: uuid.NewV7().String(),
+ }
release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim, lease)
activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{
StartedBy: &user, Step: "Backing up volumes", LatestMessage: "Creating system-managed volume backups",
diff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.go
index c98d526f1..e20567497 100644
--- a/backend/internal/system/children/backup/handler.go
+++ b/backend/internal/system/children/backup/handler.go
@@ -2,7 +2,6 @@ package backup
import (
"context"
- "errors"
"github.com/danielgtaylor/huma/v2"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
@@ -108,7 +107,7 @@ func NewHandler(service *Service, activityService *activity.ActivityService, app
func (h *Handler) ListHistory(ctx context.Context, input *ListBackupHistoryInput) (*handlerutil.Page[backup.HistoryEntry], error) {
params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
- params.Filters = map[string]string{"type": input.Type}
+ params.Filters["type"] = input.Type
history, page, err := h.service.ListBackupHistory(ctx, params)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
@@ -222,11 +221,9 @@ func (h *Handler) Create(ctx context.Context, input *CreateSystemBackupInput) (*
return nil, err
}
run, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, input.Body)
- if errors.Is(err, ErrSystemBackupAlreadyRunning) {
- return nil, huma.Error409Conflict(err.Error())
- }
if err != nil {
- return nil, huma.Error500InternalServerError(err.Error())
+ apiErr := common.ToAPIError(err)
+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)
}
return &SystemBackupOutput{Body: *run}, nil
}
@@ -274,14 +271,9 @@ func (h *Handler) RestoreFiles(ctx context.Context, input *RestoreSystemBackupFi
}, func(activityCtx context.Context) error {
return h.service.RestoreBackupFiles(activityCtx, input.ID, input.Body, *user)
})
- if errors.Is(err, ErrSystemBackupAlreadyRunning) {
- return nil, huma.Error409Conflict(err.Error())
- }
- if errors.Is(err, common.ErrBadRequest) {
- return nil, huma.Error400BadRequest(err.Error())
- }
if err != nil {
- return nil, huma.Error500InternalServerError(err.Error())
+ apiErr := common.ToAPIError(err)
+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)
}
return handlerutil.MessageOutput("Arcane project files restored successfully", activityID), nil
}
diff --git a/backend/internal/system/children/backup/module.go b/backend/internal/system/children/backup/module.go
index b926b2882..a2baedbd7 100644
--- a/backend/internal/system/children/backup/module.go
+++ b/backend/internal/system/children/backup/module.go
@@ -4,6 +4,7 @@ import (
"net/http"
"github.com/danielgtaylor/huma/v2"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup/children/volumes"
@@ -211,9 +212,5 @@ func RegisterRoutes(api huma.API, h *Handler) {
h.Delete,
)
- var volumeService *volumes.Service
- if h.service != nil {
- volumeService = h.service.volumes
- }
- volumes.RegisterRoutes(api, volumes.NewHandler(volumeService, h.appCtx))
+ volumes.RegisterRoutes(api, volumes.NewHandler(kit.FromPtr(h.service).volumes, h.appCtx))
}
diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.go
index 15ace75d9..cb3522171 100644
--- a/backend/internal/system/children/backup/service.go
+++ b/backend/internal/system/children/backup/service.go
@@ -66,7 +66,7 @@ const (
systemRecoveryHelperPath = "/app/arcane-recovery-helper"
)
-var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")
+var ErrSystemBackupAlreadyRunning = common.Classify(common.ErrConflict, errors.New("an Arcane system backup is already running"))
// Store persists system backup runs and policies through their public DTOs;
// the system domain owns the persistence models.
@@ -167,40 +167,53 @@ type preparedSystemBackup struct {
}
func NewService(deps Dependencies) *Service {
- service := &Service{
- store: deps.Store, dockerService: deps.Docker, volumeService: deps.Volumes, engine: deps.Engine,
- s3Destinations: deps.S3Destinations, activityService: deps.Activity, settingsService: deps.Settings, config: deps.Config,
- recoveryKeys: deps.RecoveryKeys, resolveRuntimeOptions: deps.ResolveRuntimeOptions,
- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
- }
- var counterErr error
- service.operations, counterErr = otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",
+ operations, counterErr := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",
metric.WithDescription("Backup operations by outcome"), metric.WithUnit("{backup}"))
if counterErr != nil {
otel.Handle(counterErr)
}
- service.snapshots = snapshots.NewService(
- deps.Engine,
- deps.SQLDB,
- deps.ActorStore,
- service.localRepository,
- service.remoteRepository,
- service.databaseFile,
- func(ctx context.Context) string {
- return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")
- },
- service.recoveryEnvironment,
- )
- service.volumes = volumes.NewService(volumes.Dependencies{
- DB: deps.DB,
- Engine: deps.Engine,
- Volumes: deps.Volumes,
- S3Destinations: deps.S3Destinations,
- Activity: deps.Activity,
- Settings: deps.Settings,
- Jobs: service.jobs,
- AlreadyRunning: ErrSystemBackupAlreadyRunning,
- })
+ jobs := entityjobs.New("system-backup:", backup.SystemAdmissionScope)
+ var service *Service
+ service = &Service{
+ store: deps.Store,
+ dockerService: deps.Docker,
+ volumeService: deps.Volumes,
+ engine: deps.Engine,
+ s3Destinations: deps.S3Destinations,
+ activityService: deps.Activity,
+ settingsService: deps.Settings,
+ config: deps.Config,
+ recoveryKeys: deps.RecoveryKeys,
+ resolveRuntimeOptions: deps.ResolveRuntimeOptions,
+ jobs: jobs,
+ operations: operations,
+ snapshots: snapshots.NewService(
+ deps.Engine,
+ deps.SQLDB,
+ deps.ActorStore,
+ func(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
+ return service.localRepository(ctx, dockerClient, readOnly)
+ },
+ func(ctx context.Context, destinationID string) (backup.Repository, error) {
+ return service.remoteRepository(ctx, destinationID)
+ },
+ func() (string, error) { return service.databaseFile() },
+ func(ctx context.Context) string {
+ return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")
+ },
+ func(ctx context.Context) map[string]string { return service.recoveryEnvironment(ctx) },
+ ),
+ volumes: volumes.NewService(volumes.Dependencies{
+ DB: deps.DB,
+ Engine: deps.Engine,
+ Volumes: deps.Volumes,
+ S3Destinations: deps.S3Destinations,
+ Activity: deps.Activity,
+ Settings: deps.Settings,
+ Jobs: jobs,
+ AlreadyRunning: ErrSystemBackupAlreadyRunning,
+ }),
+ }
return service
}
@@ -274,29 +287,40 @@ func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {
func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
var err error
s.flow = engine
- template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceName: new("Arcane")}
s.backupWorkflow, err = engine.Define(flow.Definition{
Name: "system-backup",
Version: 1,
Fingerprint: "9b44e4c38b7fcf8951468cc43a51b66c644d1104f1a56fa49bf8c065dc51c046",
Concurrency: 1,
Timeout: 24 * time.Hour,
- Activity: template,
- Labels: map[string]string{"backup": "Creating system backup"},
- Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeResourceAction,
+ EnvironmentID: "0",
+ ResourceType: new("system_backup"),
+ ResourceName: new("Arcane"),
+ },
+ Labels: map[string]string{"backup": "Creating system backup"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},
})
if err != nil {
return err
}
- template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Creating scheduled system backup", "Creating scheduled Arcane system backup"
s.policyWorkflow, err = engine.Define(flow.Definition{
Name: "system-backup-policy",
Version: 1,
Fingerprint: "42fa9de20fefac192148a5f6d68132e34ae87948160b5bb0cc5876d506843bcf",
Concurrency: 1,
Timeout: 24 * time.Hour,
- Activity: template,
- Labels: map[string]string{"backup": "Creating scheduled system backup"},
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeResourceAction,
+ EnvironmentID: "0",
+ ResourceType: new("system_backup"),
+ ResourceName: new("Arcane"),
+ StartedBy: &usertypes.SystemUser,
+ Step: "Creating scheduled system backup",
+ LatestMessage: "Creating scheduled Arcane system backup",
+ },
+ Labels: map[string]string{"backup": "Creating scheduled system backup"},
Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
var policyID string
if decodeErr := t.Payload(&policyID); decodeErr != nil {
@@ -402,58 +426,87 @@ func (s *Service) prepareBackup(ctx context.Context, trigger string, request bac
if err != nil {
return nil, err
}
- var checkpoint systemBackupRecovery
- destinationID := strings.TrimSpace(request.S3DestinationID)
- if request.PolicyID != "" {
- policy, policyErr := s.store.Policy(ctx, request.PolicyID)
- if policyErr != nil {
- return nil, policyErr
- }
- if policy == nil {
- return nil, errors.New("system backup policy not found")
+ selected, err := func() (*backuptypes.SystemBackupPolicy, error) {
+ if request.PolicyID != "" {
+ policy, policyErr := s.store.Policy(ctx, request.PolicyID)
+ if policyErr == nil && policy == nil {
+ return nil, errors.New("system backup policy not found")
+ }
+ return policy, policyErr
}
- checkpoint.LocalEnabled, checkpoint.S3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
- } else {
switch request.Destination {
case backuptypes.SystemBackupDestinationLocal, "":
- checkpoint.LocalEnabled = true
+ return &backuptypes.SystemBackupPolicy{
+ LocalEnabled: true,
+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),
+ }, nil
case backuptypes.SystemBackupDestinationS3:
- checkpoint.S3Enabled = true
+ return &backuptypes.SystemBackupPolicy{
+ S3Enabled: true,
+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),
+ }, nil
case backuptypes.SystemBackupDestinationLocalS3:
- checkpoint.LocalEnabled, checkpoint.S3Enabled = true, true
+ return &backuptypes.SystemBackupPolicy{
+ LocalEnabled: true,
+ S3Enabled: true,
+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),
+ }, nil
default:
return nil, errors.New("unknown system backup destination")
}
+ }()
+ if err != nil {
+ return nil, err
}
- if !checkpoint.LocalEnabled && !checkpoint.S3Enabled {
+ if !selected.LocalEnabled && !selected.S3Enabled {
return nil, errors.New("select at least one system backup destination")
}
- if checkpoint.S3Enabled && destinationID == "" {
+ if selected.S3Enabled && selected.S3DestinationID == "" {
return nil, errors.New("select an S3 destination for the system backup")
}
- destination := backuptypes.SystemBackupDestinationS3
- switch {
- case checkpoint.LocalEnabled && checkpoint.S3Enabled:
- destination = backuptypes.SystemBackupDestinationLocalS3
- case checkpoint.LocalEnabled:
- destination = backuptypes.SystemBackupDestinationLocal
- }
run := &backuptypes.SystemBackupRun{
- ID: "system-" + uuid.New().String(), CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,
- Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID,
+ ID: "system-" + uuid.New().String(),
+ CreatedAt: time.Now().UTC(),
+ Status: backuptypes.SystemBackupStatusRunning,
+ Trigger: trigger,
+ Destination: func() backuptypes.SystemBackupDestination {
+ switch {
+ case selected.LocalEnabled && selected.S3Enabled:
+ return backuptypes.SystemBackupDestinationLocalS3
+ case selected.LocalEnabled:
+ return backuptypes.SystemBackupDestinationLocal
+ default:
+ return backuptypes.SystemBackupDestinationS3
+ }
+ }(),
+ S3DestinationID: selected.S3DestinationID,
+ PolicyID: request.PolicyID,
}
if createErr := s.store.CreateRun(ctx, run); createErr != nil {
return nil, createErr
}
- checkpoint.BackupID = run.ID
+ checkpoint := systemBackupRecovery{
+ BackupID: run.ID,
+ LocalEnabled: selected.LocalEnabled,
+ S3Enabled: selected.S3Enabled,
+ }
recoveryData, err := json.Marshal(checkpoint)
if err == nil {
- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: recoveryData})
+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: "system_backup",
+ ID: cmp.Or(run.PolicyID, run.ID),
+ Status: scheduler.Running,
+ RecoveryData: recoveryData,
+ })
}
if err != nil {
return nil, err
}
- return &preparedSystemBackup{run: run, recoveryKey: recoveryKey, checkpoint: checkpoint}, nil
+ return &preparedSystemBackup{
+ run: run,
+ recoveryKey: recoveryKey,
+ checkpoint: checkpoint,
+ }, nil
}
// executeBackup snapshots straight to S3 for S3-only runs, otherwise keeps a local snapshot and optionally replicates it to S3, then records the run's result.
@@ -465,21 +518,42 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBac
attribute.String("arcane.backup.s3_destination_id", run.S3DestinationID))
}()
defer func() {
- if err = cmp.Or(err, ctx.Err()); err != nil {
- run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()
- } else {
- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
+ err = cmp.Or(err, ctx.Err())
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: kit.Ternary(err != nil, backuptypes.SystemBackupStatusFailed, backuptypes.SystemBackupStatusSucceeded),
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: kit.Ternary(err != nil, fmt.Sprint(err), ""),
}
if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {
err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
}
if err == nil {
- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})
+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: "system_backup",
+ ID: cmp.Or(run.PolicyID, run.ID),
+ Status: scheduler.Succeeded,
+ })
}
}()
defer utils.RecoverToError(&err, "system backup")
progress := func(stage string, status scheduler.RunStatus, message string) error {
- return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":" + stage, Status: status, Message: message})
+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: "backup_destination",
+ ID: run.ID + ":" + stage,
+ Status: status,
+ Message: message,
+ })
}
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
@@ -489,28 +563,57 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBac
return run, progressErr
}
// S3-only runs write straight to S3 so no local repository is created.
- s3Only := !checkpoint.LocalEnabled
- var repository backup.Repository
- if s3Only {
- repository, err = s.remoteRepository(ctx, run.S3DestinationID)
- } else {
- repository, err = s.localRepository(ctx, dockerClient, false)
- }
- var staged backup.Snapshot
- if err == nil {
- staged, err = s.snapshots.Create(ctx, dockerClient, repository, key, run.ID)
+ repository, err := func() (backup.Repository, error) {
+ if !checkpoint.LocalEnabled {
+ return s.remoteRepository(ctx, run.S3DestinationID)
+ }
+ return s.localRepository(ctx, dockerClient, false)
+ }()
+ if err != nil {
+ return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(checkpoint.LocalEnabled, "local", "S3"), err)
}
+ staged, err := s.snapshots.Create(ctx, dockerClient, repository, key, run.ID)
if err != nil {
- return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(s3Only, "S3", "local"), err)
+ return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(checkpoint.LocalEnabled, "local", "S3"), err)
}
if progressErr := progress("stage", scheduler.Succeeded, staged.ID); progressErr != nil {
return run, progressErr
}
- if s3Only {
- run.RemoteSnapshotID, run.Size = staged.ID, staged.Size
+ if !checkpoint.LocalEnabled {
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: staged.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: staged.ID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
return run, s.store.SaveRun(ctx, run)
}
- run.LocalSnapshotID, run.Size = staged.ID, staged.Size
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: staged.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: staged.ID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
return run, saveErr
}
@@ -531,7 +634,22 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBac
if err != nil {
return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err)
}
- run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: cmp.Or(run.Size, snapshot.Size),
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: snapshot.ID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
return run, saveErr
}
@@ -545,11 +663,26 @@ func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParams
}
names := s.s3Destinations.DestinationsByID(ctx)
remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)
- for i := range result {
- result[i].S3DestinationName = names[result[i].S3DestinationID].Name
- result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, "arcane-system-recovery", result[i].RemoteSnapshotID)
- }
- return result, response, nil
+ decorated := make([]backuptypes.SystemBackupRun, 0, len(result))
+ for _, run := range result {
+ decorated = append(decorated, backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: remoteAvailable(run.S3DestinationID, "arcane-system-recovery", run.RemoteSnapshotID),
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: names[run.S3DestinationID].Name,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ })
+ }
+ return decorated, response, nil
}
// BrowseBackupFiles returns one page from the backup's logical project tree.
@@ -643,12 +776,14 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst
}
}
// A run without snapshots (a failed attempt) is just a row; deleting it needs neither the key nor Rustic.
- key := ""
- if len(localRuns) > 0 || len(remoteGroups) > 0 {
- var err error
- if key, err = s.recoveryKey(ctx, recoveryKey); err != nil {
- return err
+ key, err := func() (string, error) {
+ if len(localRuns) == 0 && len(remoteGroups) == 0 {
+ return "", nil
}
+ return s.recoveryKey(ctx, recoveryKey)
+ }()
+ if err != nil {
+ return err
}
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
@@ -664,12 +799,65 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst
deleteErr = fmt.Errorf("failed to delete local snapshots: %w", repoErr)
} else {
for _, run := range localRuns {
- run.LocalSnapshotID = ""
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: "",
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
}
}
}
for destinationID, group := range remoteGroups {
- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshots(ctx, dockerClient, key, destinationID, group))
+ snapshotIDs := make([]string, len(group))
+ for index, run := range group {
+ snapshotIDs[index] = run.RemoteSnapshotID
+ }
+ repository, repoErr := s.remoteRepository(ctx, destinationID)
+ if repoErr == nil {
+ observation, checkErr := backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")
+ switch {
+ case checkErr != nil:
+ repoErr = checkErr
+ case !observation.Available:
+ // Nothing remains to forget; drop the references so the run can be removed.
+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))
+ default:
+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)
+ }
+ }
+ if repoErr != nil {
+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete S3 snapshots: %w", repoErr))
+ continue
+ }
+ for _, run := range group {
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: "",
+ S3DestinationID: "",
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
+ }
}
for _, run := range backupRuns {
if run.LocalSnapshotID == "" && run.RemoteSnapshotID == "" {
@@ -678,13 +866,30 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst
}
continue
}
- switch {
- case run.LocalSnapshotID != "" && run.RemoteSnapshotID != "":
- run.Destination = backuptypes.SystemBackupDestinationLocalS3
- case run.LocalSnapshotID != "":
- run.Destination = backuptypes.SystemBackupDestinationLocal
- default:
- run.Destination = backuptypes.SystemBackupDestinationS3
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: func() backuptypes.SystemBackupDestination {
+ switch {
+ case run.LocalSnapshotID != "" && run.RemoteSnapshotID != "":
+ return backuptypes.SystemBackupDestinationLocalS3
+ case run.LocalSnapshotID != "":
+ return backuptypes.SystemBackupDestinationLocal
+ default:
+ return backuptypes.SystemBackupDestinationS3
+ }
+ }(),
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
}
if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
deleteErr = errors.Join(deleteErr, saveErr)
@@ -693,34 +898,6 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst
return deleteErr
}
-// forgetRemoteSnapshots forgets one S3 destination's snapshots and drops the runs' references to them.
-func (s *Service) forgetRemoteSnapshots(ctx context.Context, dockerClient *client.Client, key, destinationID string, group []*backuptypes.SystemBackupRun) error {
- snapshotIDs := make([]string, len(group))
- for index, run := range group {
- snapshotIDs[index] = run.RemoteSnapshotID
- }
- repository, repoErr := s.remoteRepository(ctx, destinationID)
- var observation backuptypes.RepositoryObservation
- if repoErr == nil {
- observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")
- }
- switch {
- case repoErr != nil:
- case !observation.Available:
- // Nothing remains to forget; drop the references so the run can be removed.
- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))
- default:
- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)
- }
- if repoErr != nil {
- return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)
- }
- for _, run := range group {
- run.RemoteSnapshotID, run.S3DestinationID = "", ""
- }
- return nil
-}
-
// PruneLocalRepository frees the data of deleted local system backups once Rustic's keep-delete window passes.
func (s *Service) PruneLocalRepository(ctx context.Context) error {
key, err := s.recoveryKeys.Get(ctx)
@@ -820,8 +997,22 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptyp
if err != nil {
return nil, fmt.Errorf("failed to upload system recovery snapshot: %w", err)
}
- run.RemoteSnapshotID, run.S3DestinationID = snapshot.ID, request.S3DestinationID
- run.Destination = backuptypes.SystemBackupDestinationLocalS3
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: backuptypes.SystemBackupDestinationLocalS3,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: snapshot.ID,
+ S3DestinationID: request.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
if saveUploadedBackupErr := s.store.SaveRun(ctx, run); saveUploadedBackupErr != nil {
return nil, fmt.Errorf("failed to save uploaded system backup: %w", saveUploadedBackupErr)
}
@@ -882,26 +1073,35 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if err != nil {
return err
}
- // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.
- rusticNetwork := ""
- if current.NetworkSettings != nil {
- names := slices.Sorted(maps.Keys(current.NetworkSettings.Networks))
- if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {
- rusticNetwork = names[index]
- }
- }
- request := recovery.RestoreRequest{
- BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: plan.SnapshotID, RecoveryKey: key,
- LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,
- Size: run.Size, NetworkMode: rusticNetwork,
+ requestData, err := json.Marshal(recovery.RestoreRequest{
+ BackupID: run.ID,
+ ContainerID: current.ID,
+ ContainerImage: current.Config.Image,
+ SnapshotID: plan.SnapshotID,
+ RecoveryKey: key,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ Size: run.Size,
+ // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.
+ NetworkMode: func() string {
+ names := slices.Sorted(maps.Keys(kit.FromPtr(current.NetworkSettings).Networks))
+ if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {
+ return names[index]
+ }
+ return ""
+ }(),
SafetyBackup: &recovery.SafetyBackup{
- ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,
- Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,
+ ID: safetyBackup.ID,
+ LocalSnapshotID: safetyBackup.LocalSnapshotID,
+ Size: safetyBackup.Size,
+ CreatedAt: safetyBackup.CreatedAt,
},
- ProjectsSetting: s.projectsSetting(ctx), ProjectsIncluded: plan.ProjectsIncluded,
- Stages: plan.Stages, RollbackStages: rollbackStages,
- }
- requestData, err := json.Marshal(request)
+ ProjectsSetting: s.projectsSetting(ctx),
+ ProjectsIncluded: plan.ProjectsIncluded,
+ Stages: plan.Stages,
+ RollbackStages: rollbackStages,
+ })
if err != nil {
return fmt.Errorf("encode recovery request: %w", err)
}
@@ -934,27 +1134,41 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if err != nil {
return fmt.Errorf("resolve Arcane executable: %w", err)
}
- helperExecutable, mounts := executablePath, append(slices.Clone(runtimeMounts), *appDataMount)
- if executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath); executableMount != nil {
- if executableMount.Type != mount.TypeBind {
- return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")
- }
- executableMount.ReadOnly = true
- helperExecutable, mounts = systemRecoveryHelperPath, append(mounts, *executableMount)
- }
- hostConfig := &container.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}
- if current.HostConfig != nil {
- hostConfig.SecurityOpt = append([]string{}, current.HostConfig.SecurityOpt...)
- hostConfig.Privileged = current.HostConfig.Privileged
+ executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath)
+ if executableMount != nil && executableMount.Type != mount.TypeBind {
+ return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")
+ }
+ mounts := append(slices.Clone(runtimeMounts), *appDataMount)
+ if executableMount != nil {
+ mounts = append(mounts, mount.Mount{
+ Type: executableMount.Type,
+ Source: executableMount.Source,
+ Target: executableMount.Target,
+ ReadOnly: true,
+ Consistency: executableMount.Consistency,
+ BindOptions: executableMount.BindOptions,
+ VolumeOptions: executableMount.VolumeOptions,
+ ImageOptions: executableMount.ImageOptions,
+ TmpfsOptions: executableMount.TmpfsOptions,
+ ClusterOptions: executableMount.ClusterOptions,
+ })
}
created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &container.Config{
- Image: current.Config.Image, Cmd: []string{helperExecutable, "recovery-restore", "--request", requestFile}, User: "0:0",
+ Image: current.Config.Image,
+ Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},
+ User: "0:0",
Env: containerEnv,
Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},
},
- HostConfig: hostConfig,
- Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),
+ HostConfig: &container.HostConfig{
+ AutoRemove: true,
+ Mounts: mounts,
+ NetworkMode: networkMode,
+ SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),
+ Privileged: kit.FromPtr(current.HostConfig).Privileged,
+ },
+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),
})
if err != nil {
return fmt.Errorf("create recovery helper container: %w", err)
@@ -1005,10 +1219,8 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*back
if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {
return nil, setErr
}
- if s.volumeService != nil {
- if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {
- slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())
- }
+ if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {
+ slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())
}
return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil
}
@@ -1022,24 +1234,49 @@ func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPol
if err != nil {
return nil, err
}
- result := &backuptypes.SystemBackupPolicyCollection{
- Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,
- }
destinations := s.s3Destinations.DestinationsByID(ctx)
- for i := range policies {
- lastRun, runErr := s.store.LatestPolicyRun(ctx, policies[i].ID)
+ decorated := make([]backuptypes.SystemBackupPolicy, 0, len(policies))
+ for _, policy := range policies {
+ lastRun, runErr := s.store.LatestPolicyRun(ctx, policy.ID)
if runErr != nil {
return nil, fmt.Errorf("failed to load latest system backup: %w", runErr)
}
- if lastRun != nil {
- lastRun.S3DestinationName = destinations[lastRun.S3DestinationID].Name
- }
- dto := policies[i]
- dto.LastRun = lastRun
- dto.S3DestinationName = destinations[dto.S3DestinationID].Name
- result.Policies = append(result.Policies, dto)
- }
- return result, nil
+ decorated = append(decorated, backuptypes.SystemBackupPolicy{ //exhaustruct:enforce
+ ID: policy.ID,
+ Enabled: policy.Enabled,
+ Schedule: policy.Schedule,
+ RetentionCount: policy.RetentionCount,
+ LocalEnabled: policy.LocalEnabled,
+ S3Enabled: policy.S3Enabled,
+ S3DestinationID: policy.S3DestinationID,
+ S3DestinationName: destinations[policy.S3DestinationID].Name,
+ LastRun: func() *backuptypes.SystemBackupRun {
+ if lastRun == nil {
+ return nil
+ }
+ return &backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: lastRun.RemoteAvailable,
+ ActivityID: lastRun.ActivityID,
+ ID: lastRun.ID,
+ Size: lastRun.Size,
+ CreatedAt: lastRun.CreatedAt,
+ Status: lastRun.Status,
+ Trigger: lastRun.Trigger,
+ Destination: lastRun.Destination,
+ LocalSnapshotID: lastRun.LocalSnapshotID,
+ RemoteSnapshotID: lastRun.RemoteSnapshotID,
+ S3DestinationID: lastRun.S3DestinationID,
+ S3DestinationName: destinations[lastRun.S3DestinationID].Name,
+ PolicyID: lastRun.PolicyID,
+ Error: lastRun.Error,
+ }
+ }(),
+ })
+ }
+ return &backuptypes.SystemBackupPolicyCollection{
+ Policies: decorated,
+ RecoveryKeyStored: configured,
+ }, nil
}
func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {
@@ -1052,15 +1289,23 @@ func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.Upda
if normalizeErr != nil {
return normalizeErr
}
- update = normalized
- if update.Enabled && !configured {
+ if normalized.Enabled && !configured {
return errors.New("configure a recovery key before enabling scheduled system backups")
}
- if update.Enabled && !s.SupportedDatabaseProvider() {
+ if normalized.Enabled && !s.SupportedDatabaseProvider() {
return errors.New("scheduled system backups require the SQLite database provider")
}
- policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount
- policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationID
+ *policy = backuptypes.SystemBackupPolicy{ //exhaustruct:enforce
+ ID: policy.ID,
+ Enabled: normalized.Enabled,
+ Schedule: normalized.Schedule,
+ RetentionCount: normalized.RetentionCount,
+ LocalEnabled: normalized.LocalEnabled,
+ S3Enabled: normalized.S3Enabled,
+ S3DestinationID: normalized.S3DestinationID,
+ S3DestinationName: policy.S3DestinationName,
+ LastRun: policy.LastRun,
+ }
return nil
}
if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicy); runErr != nil {
@@ -1083,10 +1328,12 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (sche
if policy == nil || !policy.Enabled {
return scheduler.Outcome{Status: scheduler.Skipped}, nil
}
- var remoteErr error
- if policy.S3Enabled {
- remoteErr = s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)
- }
+ remoteErr := func() error {
+ if !policy.S3Enabled {
+ return nil
+ }
+ return s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)
+ }()
remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {
return s.store.DisablePolicyRemote(ctx, policy, field)
})
@@ -1133,17 +1380,19 @@ func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.
}
if current != nil && current.RetentionCount > 0 {
if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {
- outcome.Status = scheduler.Partial
- return outcome, applyRetentionErr
+ return scheduler.Outcome{
+ Status: scheduler.Partial,
+ Message: outcome.Message,
+ ActivityID: outcome.ActivityID,
+ Targets: outcome.Targets,
+ Steps: outcome.Steps,
+ }, applyRetentionErr
}
}
return outcome, nil
}
func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {
- if policy == nil {
- return
- }
if !policy.Enabled {
s.jobs.Unregister(ctx, policy.ID)
return
@@ -1157,9 +1406,13 @@ func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *back
Activity: activitylib.StartRequest{
ResourceID: new(policyID),
Metadata: database.JSON{
- "action": "scheduled_system_backup", "policyId": policyID, "schedule": policy.Schedule,
- "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,
- "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,
+ "action": "scheduled_system_backup",
+ "policyId": policyID,
+ "schedule": policy.Schedule,
+ "retentionCount": policy.RetentionCount,
+ "localEnabled": policy.LocalEnabled,
+ "s3Enabled": policy.S3Enabled,
+ "s3DestinationId": policy.S3DestinationID,
},
},
ScheduleFn: func(ctx context.Context) string {
@@ -1207,14 +1460,7 @@ func (s *Service) applyRetention(ctx context.Context, policyID string, keep int,
// projectsSetting returns the effective projectsDirectory setting.
func (s *Service) projectsSetting(ctx context.Context) string {
- fallback := ""
- if s.config != nil {
- fallback = s.config.ProjectsDirectory
- }
- if s.settingsService == nil {
- return fallback
- }
- return s.settingsService.GetStringSetting(ctx, "projectsDirectory", fallback)
+ return s.settingsService.GetStringSetting(ctx, "projectsDirectory", s.config.ProjectsDirectory)
}
// StartBackup prepares a manual backup, then submits its workflow.
@@ -1241,10 +1487,21 @@ func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, request
return nil, errors.Join(holdErr, s.store.FailRun(context.WithoutCancel(ctx), run.ID, holdErr.Error()))
}
}
- input := manualBackupInput{Checkpoint: prepared.checkpoint, Requester: requester}
- activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{
- ResourceID: new("arcane"), StartedBy: &user, Step: "Creating system backup", LatestMessage: "Creating Arcane system backup",
- Metadata: database.JSON{"action": "create_system_backup", "backupId": run.ID, "destination": run.Destination, "s3DestinationId": run.S3DestinationID, "policyId": run.PolicyID},
+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, manualBackupInput{
+ Checkpoint: prepared.checkpoint,
+ Requester: requester,
+ }, activitylib.StartRequest{
+ ResourceID: new("arcane"),
+ StartedBy: &user,
+ Step: "Creating system backup",
+ LatestMessage: "Creating Arcane system backup",
+ Metadata: database.JSON{
+ "action": "create_system_backup",
+ "backupId": run.ID,
+ "destination": run.Destination,
+ "s3DestinationId": run.S3DestinationID,
+ "policyId": run.PolicyID,
+ },
}, backup.AcceptedTarget(run.ID))
if err != nil {
release(ctx)
@@ -1256,9 +1513,22 @@ func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, request
s.watchBackup(background, run.ID, activityID)
release(background)
}()
- dto := *run
- dto.ActivityID = activityID
- return &dto, nil
+ return &backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: activityID,
+ ID: run.ID,
+ Size: run.Size,
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: run.RemoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }, nil
}
// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.
@@ -1310,11 +1580,21 @@ func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, t
return outcome, err
}
confirm := func() (scheduler.Outcome, error) {
- target.Status = scheduler.Succeeded
- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ confirmed := scheduler.TargetOutcome{ //exhaustruct:enforce
+ RecoveryData: target.RecoveryData,
+ ResourceType: target.ResourceType,
+ ID: target.ID,
+ Status: scheduler.Succeeded,
+ Message: target.Message,
+ ActivityID: target.ActivityID,
+ }
+ if progressErr := jobcontext.Progress(ctx, confirmed); progressErr != nil {
return outcome, progressErr
}
- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
+ return scheduler.Outcome{
+ Status: scheduler.Succeeded,
+ Targets: []scheduler.TargetOutcome{confirmed},
+ }, nil
}
run, err := s.store.Run(ctx, checkpoint.BackupID)
if err != nil {
@@ -1347,28 +1627,45 @@ func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, t
}
if staged.ID == "" && run.RemoteSnapshotID == "" {
// Nothing survived the interruption, so the backup runs again.
- if _, err = s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: checkpoint}); err != nil {
+ if _, err = s.executeBackup(ctx, &preparedSystemBackup{
+ run: run,
+ recoveryKey: key,
+ checkpoint: checkpoint,
+ }); err != nil {
return outcome, err
}
return confirm()
}
- if checkpoint.LocalEnabled {
- run.LocalSnapshotID, run.Size = staged.ID, staged.Size
- }
// A missing remote snapshot implies a staged one; the rerun above handled neither surviving.
+ remoteSnapshotID := run.RemoteSnapshotID
if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {
snapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
if replicateErr != nil {
return outcome, replicateErr
}
- run.RemoteSnapshotID = snapshot.ID
+ remoteSnapshotID = snapshot.ID
}
if !checkpoint.LocalEnabled && staged.ID != "" {
if forgetErr := s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {
return outcome, forgetErr
}
}
- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: kit.Ternary(checkpoint.LocalEnabled, staged.Size, run.Size),
+ CreatedAt: run.CreatedAt,
+ Status: backuptypes.SystemBackupStatusSucceeded,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: kit.Ternary(checkpoint.LocalEnabled, staged.ID, run.LocalSnapshotID),
+ RemoteSnapshotID: remoteSnapshotID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: "",
+ }
if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
return outcome, saveErr
}
@@ -1421,7 +1718,22 @@ func (s *Service) observeBackupSnapshots(
return staged, local, remote, findErr
}
if found {
- run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)
+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: run.RemoteAvailable,
+ ActivityID: run.ActivityID,
+ ID: run.ID,
+ Size: cmp.Or(run.Size, snapshot.Size),
+ CreatedAt: run.CreatedAt,
+ Status: run.Status,
+ Trigger: run.Trigger,
+ Destination: run.Destination,
+ LocalSnapshotID: run.LocalSnapshotID,
+ RemoteSnapshotID: snapshot.ID,
+ S3DestinationID: run.S3DestinationID,
+ S3DestinationName: run.S3DestinationName,
+ PolicyID: run.PolicyID,
+ Error: run.Error,
+ }
}
}
// S3-only runs snapshot straight to S3, so only runs from older versions left a local staged snapshot to recover,
@@ -1499,11 +1811,20 @@ func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, err
if err != nil {
return nil, err
}
- target := scheduler.TargetOutcome{ResourceType: "system_backup", ID: runID, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}
- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: "system_backup",
+ ID: runID,
+ Status: scheduler.Running,
+ RecoveryData: checkpoint,
+ ActivityID: t.ActivityID(),
+ }); progressErr != nil {
return nil, progressErr
}
- if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: input.Checkpoint}); executeErr != nil {
+ if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{
+ run: run,
+ recoveryKey: key,
+ checkpoint: input.Checkpoint,
+ }); executeErr != nil {
return nil, executeErr
}
return succeeded, nil
@@ -1517,23 +1838,47 @@ func (s *Service) ListBackupHistory(ctx context.Context, params pagination.Query
}
destinations := s.s3Destinations.DestinationsByID(ctx)
remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)
- instanceID := ""
- if s.settingsService != nil {
- instanceID = s.settingsService.GetSettingsConfig().InstanceID.Value
- }
- for i := range history {
- history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name
+ instanceID := s.settingsService.GetSettingsConfig().InstanceID.Value
+ decorated := make([]backuptypes.HistoryEntry, 0, len(history))
+ for _, entry := range history {
// Volume backups discovered from another instance live under that instance's root.
- root := "arcane-system-recovery"
- if history[i].ResourceType == "volume" {
- root = ""
- if owner := cmp.Or(history[i].RemoteInstanceID, instanceID); owner != "" {
- root = path.Join(backup.VolumeRoot, owner)
+ root := func() string {
+ owner := cmp.Or(entry.RemoteInstanceID, instanceID)
+ switch {
+ case entry.ResourceType != "volume":
+ return "arcane-system-recovery"
+ case owner == "":
+ return ""
+ default:
+ return path.Join(backup.VolumeRoot, owner)
}
- }
- history[i].RemoteAvailable = remoteAvailable(history[i].S3DestinationID, root, history[i].RemoteSnapshotID)
- }
-
- page.GrandTotalItems = page.TotalItems
- return history, page, nil
+ }()
+ decorated = append(decorated, backuptypes.HistoryEntry{ //exhaustruct:enforce
+ RemoteAvailable: remoteAvailable(entry.S3DestinationID, root, entry.RemoteSnapshotID),
+ ID: entry.ID,
+ Size: entry.Size,
+ CreatedAt: entry.CreatedAt,
+ Status: entry.Status,
+ Trigger: entry.Trigger,
+ Destination: entry.Destination,
+ Format: entry.Format,
+ LocalSnapshotID: entry.LocalSnapshotID,
+ RemoteSnapshotID: entry.RemoteSnapshotID,
+ RemoteInstanceID: entry.RemoteInstanceID,
+ S3DestinationID: entry.S3DestinationID,
+ S3DestinationName: destinations[entry.S3DestinationID].Name,
+ PolicyID: entry.PolicyID,
+ Error: entry.Error,
+ Type: entry.Type,
+ ResourceType: entry.ResourceType,
+ ResourceName: entry.ResourceName,
+ })
+ }
+ return decorated, pagination.Response{
+ TotalPages: page.TotalPages,
+ TotalItems: page.TotalItems,
+ CurrentPage: page.CurrentPage,
+ ItemsPerPage: page.ItemsPerPage,
+ GrandTotalItems: page.TotalItems,
+ }, nil
}
diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.go
index 3657612a3..c03bebae2 100644
--- a/backend/internal/system/children/backup/service_test.go
+++ b/backend/internal/system/children/backup/service_test.go
@@ -12,27 +12,33 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
)
-func TestProjectsSettingFallsBackToConfig(t *testing.T) {
- service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}
- require.Equal(t, "/app/data/projects:/host/projects", service.projectsSetting(t.Context()))
-}
-
func TestHistoryDestinationDecoration(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))
+ require.NoError(t, db.AutoMigrate(&s3.S3Destination{}, &settings.SettingVariable{}))
destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}
require.NoError(t, db.Create(&destination).Error)
- service := &Service{
- s3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil),
- store: Store{ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {
- return []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}, pagination.Response{}, nil
- }},
- }
+ store := &database.DB{DB: db}
+ settingsService, err := settings.NewSettingsService(t.Context(), store)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })
+ service := NewService(Dependencies{
+ S3Destinations: s3.NewS3DestinationService(store, func(context.Context, string) error { return nil }),
+ Settings: settingsService,
+ Store: Store{
+ ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {
+ return []backup.HistoryEntry{
+ {S3DestinationID: destination.ID},
+ {S3DestinationID: "missing"},
+ }, pagination.Response{}, nil
+ },
+ },
+ })
history, _, err := service.ListBackupHistory(t.Context(), pagination.QueryParams{})
require.NoError(t, err)
require.Equal(t, "Offsite", history[0].S3DestinationName)
@@ -58,7 +64,16 @@ func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {
OidcClientSecret: "oidc-secret",
FilePerm: 0o640,
}
- environment := (&Service{config: cfg}).recoveryEnvironment(t.Context())
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))
+ settingsService, err := settings.NewSettingsService(t.Context(), &database.DB{DB: db})
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })
+ environment := NewService(Dependencies{
+ Config: cfg,
+ Settings: settingsService,
+ }).recoveryEnvironment(t.Context())
require.Equal(t, "jwt-secret", environment["JWT_SECRET"])
require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])
require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])
diff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.go
index 41900a9bc..2b0ea382b 100644
--- a/backend/internal/system/children/prune/service.go
+++ b/backend/internal/system/children/prune/service.go
@@ -21,6 +21,7 @@ import (
"github.com/moby/moby/client"
"github.com/samber/mo"
"go.getarcane.app/docker/compat"
+ kit "go.getarcane.app/kit/pkg"
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -74,47 +75,54 @@ func (s *Service) PruneAll(ctx context.Context, environmentID string, req system
"buildCache", req.BuildCache,
)
- prune := s.beginSystemPruneInternal(ctx, environmentID, req)
- activityID := prune.activityID
- result := &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
+ prune := s.beginSystemPrune(ctx, environmentID, req)
if prune.started.IsAbsent() {
- slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", activityID)
- return result, false, nil
+ slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", prune.activityID)
+ return &system.PruneAllResult{
+ Success: true,
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),
+ }, false, nil
}
- defer s.finishSystemPruneInternal(environmentID)
+ defer s.finishSystemPrune(environmentID)
- ctx = s.activityService.Track(ctx, activityID)
- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "scheduled-prune", Status: scheduler.Running, ActivityID: activityID}); err != nil {
- result.Success = false
- result.Errors = append(result.Errors, err.Error())
- s.completeSystemPruneActivityInternal(ctx, activityID, result)
+ ctx = s.activityService.Track(ctx, prune.activityID)
+ if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ID: "scheduled-prune",
+ Status: scheduler.Running,
+ ActivityID: prune.activityID,
+ }); err != nil {
+ result := &system.PruneAllResult{
+ Success: false,
+ Errors: []string{err.Error()},
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),
+ }
+ s.completeSystemPruneActivity(ctx, prune.activityID, result)
return result, true, err
}
- s.runSystemPruneInternal(ctx, req, activityID, result)
-
- return result, true, nil
+ return s.runSystemPrune(ctx, req, prune.activityID), true, nil
}
func (s *Service) StartPruneAll(ctx context.Context, environmentID string, req system.PruneAllRequest) *system.PruneAllResult {
- prune := s.beginSystemPruneInternal(ctx, environmentID, req)
- activityID := prune.activityID
+ prune := s.beginSystemPrune(ctx, environmentID, req)
if prune.started.IsAbsent() {
- slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", activityID)
- return &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
+ slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", prune.activityID)
+ return &system.PruneAllResult{
+ Success: true,
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),
+ }
}
- backgroundCtx := utils.ActivityRuntimeContext(ctx, nil)
- backgroundCtx = s.activityService.Track(backgroundCtx, activityID)
-
+ backgroundCtx := s.activityService.Track(utils.ActivityRuntimeContext(ctx, nil), prune.activityID)
go func() {
- defer s.finishSystemPruneInternal(environmentID)
-
- result := &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
- s.runSystemPruneInternal(backgroundCtx, req, activityID, result)
+ defer s.finishSystemPrune(environmentID)
+ s.runSystemPrune(backgroundCtx, req, prune.activityID)
}()
- return &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
+ return &system.PruneAllResult{
+ Success: true,
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),
+ }
}
type systemPruneBeginResult struct {
@@ -122,7 +130,7 @@ type systemPruneBeginResult struct {
started mo.Option[struct{}]
}
-func (s *Service) beginSystemPruneInternal(ctx context.Context, environmentID string, req system.PruneAllRequest) systemPruneBeginResult {
+func (s *Service) beginSystemPrune(ctx context.Context, environmentID string, req system.PruneAllRequest) systemPruneBeginResult {
s.pruneMu.Lock()
defer s.pruneMu.Unlock()
@@ -130,29 +138,55 @@ func (s *Service) beginSystemPruneInternal(ctx context.Context, environmentID st
return systemPruneBeginResult{activityID: activityID, started: mo.None[struct{}]()}
}
- activityID := s.startSystemPruneActivityInternal(ctx, environmentID, req)
+ localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
+ EnvironmentID: environmentID,
+ Type: activitytypes.TypeSystemPrune,
+ ResourceType: new("system"),
+ ResourceName: new("Docker resources"),
+ Step: "Preparing prune",
+ LatestMessage: "System prune started",
+ Metadata: database.JSON{
+ "containers": req.Containers,
+ "images": req.Images,
+ "volumes": req.Volumes,
+ "networks": req.Networks,
+ "buildCache": req.BuildCache,
+ },
+ })
+ if err != nil {
+ slog.DebugContext(ctx, "failed to start system prune activity", "error", err)
+ }
+ activityID := kit.Ternary(err != nil, "", kit.FromPtr(localActivity).ID)
s.runningPrunes[environmentID] = activityID
return systemPruneBeginResult{activityID: activityID, started: mo.Some(struct{}{})}
}
-func (s *Service) finishSystemPruneInternal(environmentID string) {
+func (s *Service) finishSystemPrune(environmentID string) {
s.pruneMu.Lock()
defer s.pruneMu.Unlock()
delete(s.runningPrunes, environmentID)
}
-func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAllRequest, activityID string, result *system.PruneAllResult) {
+func (s *Service) runSystemPrune(ctx context.Context, req system.PruneAllRequest, activityID string) *system.PruneAllResult {
var mu sync.Mutex
+ var errs []string
+ addError := func(format string, err error) {
+ mu.Lock()
+ defer mu.Unlock()
+ errs = append(errs, fmt.Sprintf(format, err))
+ }
+ var containers, images, buildCache, volumes, networks system.PruneAllResult
// 1. Prune Containers first (sequential) as it may free up other resources
if req.Containers != nil && req.Containers.Mode != system.PruneContainerModeNone {
- s.appendSystemPruneActivityMessageInternal(ctx, activityID, "Pruning containers", 15)
+ s.appendSystemPruneActivityMessage(ctx, activityID, "Pruning containers", 15)
slog.InfoContext(ctx, "Pruning containers...", "mode", req.Containers.Mode, "until", req.Containers.Until)
- if err := s.pruneContainersInternal(ctx, *req.Containers, result); err != nil {
- result.Errors = append(result.Errors, fmt.Sprintf("Container pruning failed: %v", err))
- result.Success = false
+ part, err := s.pruneContainers(ctx, *req.Containers)
+ if err != nil {
+ addError("Container pruning failed: %v", err)
}
+ containers = part
}
// 2. Prune other resources in parallel
@@ -162,21 +196,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "system info worker")
- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning images", 40)
+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning images", 40)
slog.InfoContext(groupCtx, "Pruning images...", "mode", req.Images.Mode, "until", req.Images.Until)
- localResult := &system.PruneAllResult{}
- if err := s.pruneImagesInternal(groupCtx, *req.Images, localResult); err != nil {
- mu.Lock()
- result.Errors = append(result.Errors, fmt.Sprintf("Image pruning failed: %v", err))
- result.Success = false
- mu.Unlock()
- } else {
- mu.Lock()
- result.ImagesDeleted = append(result.ImagesDeleted, localResult.ImagesDeleted...)
- result.SpaceReclaimed += localResult.SpaceReclaimed
- result.ImageSpaceReclaimed += localResult.ImageSpaceReclaimed
- mu.Unlock()
+ part, err := s.pruneImages(groupCtx, *req.Images)
+ if err != nil {
+ addError("Image pruning failed: %v", err)
}
+ images = part
return nil
})
}
@@ -185,23 +211,16 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "system info worker")
- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning build cache", 45)
+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning build cache", 45)
slog.InfoContext(groupCtx, "Pruning build cache...", "mode", req.BuildCache.Mode, "until", req.BuildCache.Until)
- localResult := &system.PruneAllResult{}
- if err := s.pruneBuildCacheInternal(groupCtx, *req.BuildCache, localResult); err != nil {
+ part, err := s.pruneBuildCache(groupCtx, *req.BuildCache)
+ if err != nil {
slog.WarnContext(groupCtx, "Build cache pruning encountered an error", "error", err.Error())
// Surface the failure like every other prune type so a build cache that
// could not be reclaimed is reported instead of silently left behind.
- mu.Lock()
- result.Errors = append(result.Errors, fmt.Sprintf("Build cache pruning failed: %v", err))
- result.Success = false
- mu.Unlock()
- } else {
- mu.Lock()
- result.SpaceReclaimed += localResult.SpaceReclaimed
- result.BuildCacheSpaceReclaimed += localResult.BuildCacheSpaceReclaimed
- mu.Unlock()
+ addError("Build cache pruning failed: %v", err)
}
+ buildCache = part
return nil
})
}
@@ -210,21 +229,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "system info worker")
- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning volumes", 55)
+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning volumes", 55)
slog.InfoContext(groupCtx, "Pruning volumes...", "mode", req.Volumes.Mode)
- localResult := &system.PruneAllResult{}
- if err := s.pruneVolumesInternal(groupCtx, *req.Volumes, localResult); err != nil {
- mu.Lock()
- result.Errors = append(result.Errors, fmt.Sprintf("Volume pruning failed: %v", err))
- result.Success = false
- mu.Unlock()
- } else {
- mu.Lock()
- result.VolumesDeleted = append(result.VolumesDeleted, localResult.VolumesDeleted...)
- result.SpaceReclaimed += localResult.SpaceReclaimed
- result.VolumeSpaceReclaimed += localResult.VolumeSpaceReclaimed
- mu.Unlock()
+ part, err := s.pruneVolumes(groupCtx, *req.Volumes)
+ if err != nil {
+ addError("Volume pruning failed: %v", err)
}
+ volumes = part
return nil
})
}
@@ -233,19 +244,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "system info worker")
- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning networks", 65)
+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning networks", 65)
slog.InfoContext(groupCtx, "Pruning networks...", "mode", req.Networks.Mode, "until", req.Networks.Until)
- localResult := &system.PruneAllResult{}
- if err := s.pruneNetworksInternal(groupCtx, *req.Networks, localResult); err != nil {
- mu.Lock()
- result.Errors = append(result.Errors, fmt.Sprintf("Network pruning failed: %v", err))
- result.Success = false
- mu.Unlock()
- } else {
- mu.Lock()
- result.NetworksDeleted = append(result.NetworksDeleted, localResult.NetworksDeleted...)
- mu.Unlock()
+ part, err := s.pruneNetworks(groupCtx, *req.Networks)
+ if err != nil {
+ addError("Network pruning failed: %v", err)
}
+ networks = part
return nil
})
}
@@ -254,6 +259,20 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
slog.ErrorContext(ctx, "Prune operations failed", "error", err)
}
+ result := &system.PruneAllResult{
+ ContainersPruned: containers.ContainersPruned,
+ ImagesDeleted: images.ImagesDeleted,
+ VolumesDeleted: volumes.VolumesDeleted,
+ NetworksDeleted: networks.NetworksDeleted,
+ SpaceReclaimed: containers.SpaceReclaimed + images.SpaceReclaimed + buildCache.SpaceReclaimed + volumes.SpaceReclaimed,
+ ContainerSpaceReclaimed: containers.ContainerSpaceReclaimed,
+ ImageSpaceReclaimed: images.ImageSpaceReclaimed,
+ VolumeSpaceReclaimed: volumes.VolumeSpaceReclaimed,
+ BuildCacheSpaceReclaimed: buildCache.BuildCacheSpaceReclaimed,
+ Success: len(errs) == 0,
+ Errors: errs,
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
+ }
slog.InfoContext(
ctx,
"Selective prune operation completed",
@@ -282,37 +301,12 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAl
result.Errors,
),
)
- s.completeSystemPruneActivityInternal(ctx, activityID, result)
-}
-
-func (s *Service) startSystemPruneActivityInternal(ctx context.Context, environmentID string, req system.PruneAllRequest) string {
- if s.activityService == nil {
- return ""
- }
- localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
- EnvironmentID: environmentID,
- Type: activitytypes.TypeSystemPrune,
- ResourceType: new("system"),
- ResourceName: new("Docker resources"),
- Step: "Preparing prune",
- LatestMessage: "System prune started",
- Metadata: database.JSON{
- "containers": req.Containers,
- "images": req.Images,
- "volumes": req.Volumes,
- "networks": req.Networks,
- "buildCache": req.BuildCache,
- },
- })
- if err != nil {
- slog.DebugContext(ctx, "failed to start system prune activity", "error", err)
- return ""
- }
- return localActivity.ID
+ s.completeSystemPruneActivity(ctx, activityID, result)
+ return result
}
-func (s *Service) appendSystemPruneActivityMessageInternal(ctx context.Context, activityID, message string, progress int) {
- if s.activityService == nil || activityID == "" {
+func (s *Service) appendSystemPruneActivityMessage(ctx context.Context, activityID, message string, progress int) {
+ if activityID == "" {
return
}
if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{
@@ -325,81 +319,79 @@ func (s *Service) appendSystemPruneActivityMessageInternal(ctx context.Context,
}
}
-func (s *Service) completeSystemPruneActivityInternal(ctx context.Context, activityID string, result *system.PruneAllResult) {
- if s.activityService == nil || activityID == "" || result == nil {
+func (s *Service) completeSystemPruneActivity(ctx context.Context, activityID string, result *system.PruneAllResult) {
+ if activityID == "" {
return
}
- status := activitytypes.StatusSuccess
- message := "System prune completed"
- var errMessage *string
- if !result.Success || len(result.Errors) > 0 {
- if activitylib.CancelledByContext(ctx) {
- status = activitytypes.StatusCancelled
- message = "System prune cancelled"
- } else {
- status = activitytypes.StatusFailed
- message = "System prune completed with errors"
- errMessage = new(strings.Join(result.Errors, "; "))
+ status, message, errMessage := func() (activitytypes.Status, string, *string) {
+ switch {
+ case result.Success && len(result.Errors) == 0:
+ return activitytypes.StatusSuccess, "System prune completed", nil
+ case activitylib.CancelledByContext(ctx):
+ return activitytypes.StatusCancelled, "System prune cancelled", nil
+ default:
+ return activitytypes.StatusFailed, "System prune completed with errors", new(strings.Join(result.Errors, "; "))
}
- }
+ }()
if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {
slog.DebugContext(ctx, "failed to complete system prune activity", "activityId", activityID, "error", err)
}
}
-func (s *Service) pruneContainersInternal(ctx context.Context, options system.PruneContainersOptions, result *system.PruneAllResult) error {
+func (s *Service) pruneContainers(ctx context.Context, options system.PruneContainersOptions) (system.PruneAllResult, error) {
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
}
filterArgs := make(client.Filters)
if options.Mode == system.PruneContainerModeOlderThan {
if strings.TrimSpace(options.Until) == "" {
- return errors.New("container prune mode olderThan requires until")
+ return system.PruneAllResult{}, errors.New("container prune mode olderThan requires until")
}
filterArgs = filterArgs.Add("until", options.Until)
}
report, err := dockerClient.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})
if err != nil {
- return fmt.Errorf("failed to prune containers: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)
}
- result.ContainersPruned = report.Report.ContainersDeleted
- result.SpaceReclaimed += report.Report.SpaceReclaimed
- result.ContainerSpaceReclaimed += report.Report.SpaceReclaimed
- return nil
+ return system.PruneAllResult{
+ ContainersPruned: report.Report.ContainersDeleted,
+ SpaceReclaimed: report.Report.SpaceReclaimed,
+ ContainerSpaceReclaimed: report.Report.SpaceReclaimed,
+ }, nil
}
-func (s *Service) pruneImagesInternal(ctx context.Context, options system.PruneImagesOptions, result *system.PruneAllResult) error {
+func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOptions) (system.PruneAllResult, error) {
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
}
filterArgs := make(client.Filters)
switch options.Mode {
case system.PruneImageModeNone:
- return errors.New("image prune mode none is not allowed")
+ return system.PruneAllResult{}, errors.New("image prune mode none is not allowed")
case system.PruneImageModeDangling:
filterArgs = filterArgs.Add("dangling", "true")
case system.PruneImageModeAll:
filterArgs = filterArgs.Add("dangling", "false")
case system.PruneImageModeOlderThan:
if strings.TrimSpace(options.Until) == "" {
- return errors.New("image prune mode olderThan requires until")
+ return system.PruneAllResult{}, errors.New("image prune mode olderThan requires until")
}
filterArgs = filterArgs.Add("dangling", "false")
filterArgs = filterArgs.Add("until", options.Until)
default:
- return fmt.Errorf("unsupported image prune mode: %s", options.Mode)
+ return system.PruneAllResult{}, fmt.Errorf("unsupported image prune mode: %s", options.Mode)
}
report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})
if err != nil {
- return fmt.Errorf("failed to prune images: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)
}
slog.InfoContext(ctx, "Image pruning completed", "imagesDeleted", len(report.Report.ImagesDeleted), "bytesReclaimed", report.Report.SpaceReclaimed)
@@ -418,117 +410,113 @@ func (s *Service) pruneImagesInternal(ctx context.Context, options system.PruneI
slog.WarnContext(ctx, "Failed to delete image update records", "count", len(idsToDelete), "error", deleteRecordsForImagesErr.Error())
}
- result.ImagesDeleted = idsToDelete
- result.SpaceReclaimed += report.Report.SpaceReclaimed
- result.ImageSpaceReclaimed += report.Report.SpaceReclaimed
- return nil
+ return system.PruneAllResult{
+ ImagesDeleted: idsToDelete,
+ SpaceReclaimed: report.Report.SpaceReclaimed,
+ ImageSpaceReclaimed: report.Report.SpaceReclaimed,
+ }, nil
}
-func (s *Service) pruneBuildCacheInternal(ctx context.Context, options system.PruneBuildCacheOptions, result *system.PruneAllResult) error {
+func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuildCacheOptions) (system.PruneAllResult, error) {
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- result.Errors = append(result.Errors, fmt.Errorf("build cache pruning failed (connection): %w", err).Error())
slog.ErrorContext(ctx, "Error connecting to Docker for build cache prune", "error", err.Error())
- return fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)
- }
-
- pruneOptions := client.BuildCachePruneOptions{
- All: options.Mode == system.PruneBuildCacheModeAll,
+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)
}
- if options.Mode == system.PruneBuildCacheModeOlderThan {
- if strings.TrimSpace(options.Until) == "" {
- return errors.New("build cache prune mode olderThan requires until")
- }
- pruneOptions.Filters = make(client.Filters)
- pruneOptions.Filters = pruneOptions.Filters.Add("until", options.Until)
+ if options.Mode == system.PruneBuildCacheModeOlderThan && strings.TrimSpace(options.Until) == "" {
+ return system.PruneAllResult{}, errors.New("build cache prune mode olderThan requires until")
}
slog.DebugContext(ctx, "starting build cache pruning", "mode", options.Mode, "until", options.Until)
- report, err := dockerClient.BuildCachePrune(ctx, pruneOptions)
+ report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{
+ All: options.Mode == system.PruneBuildCacheModeAll,
+ Filters: kit.Ternary(options.Mode == system.PruneBuildCacheModeOlderThan, make(client.Filters).Add("until", options.Until), nil),
+ })
if err != nil {
- result.Errors = append(result.Errors, fmt.Errorf("build cache pruning failed: %w", err).Error())
slog.ErrorContext(ctx, "Error pruning build cache", "error", err.Error())
- return fmt.Errorf("failed to prune build cache: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune build cache: %w", err)
}
slog.InfoContext(ctx, "build cache pruning completed", "cacheEntriesDeleted", len(report.Report.CachesDeleted), "bytesReclaimed", report.Report.SpaceReclaimed)
- result.SpaceReclaimed += report.Report.SpaceReclaimed
- result.BuildCacheSpaceReclaimed += report.Report.SpaceReclaimed
- return nil
+ return system.PruneAllResult{
+ SpaceReclaimed: report.Report.SpaceReclaimed,
+ BuildCacheSpaceReclaimed: report.Report.SpaceReclaimed,
+ }, nil
}
-func (s *Service) pruneVolumesInternal(ctx context.Context, options system.PruneVolumesOptions, result *system.PruneAllResult) error {
- allVolumes := options.Mode == system.PruneVolumeModeAll
- report, err := s.volumeService.PruneVolumesWithOptions(ctx, allVolumes)
+func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesOptions) (system.PruneAllResult, error) {
+ report, err := s.volumeService.PruneVolumesWithOptions(ctx, options.Mode == system.PruneVolumeModeAll)
if err != nil {
- return err
+ return system.PruneAllResult{}, err
}
slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(report.VolumesDeleted), "spaceReclaimed", report.SpaceReclaimed)
- result.VolumesDeleted = report.VolumesDeleted
- result.SpaceReclaimed += report.SpaceReclaimed
- result.VolumeSpaceReclaimed += report.SpaceReclaimed
- return nil
+ return system.PruneAllResult{
+ VolumesDeleted: report.VolumesDeleted,
+ SpaceReclaimed: report.SpaceReclaimed,
+ VolumeSpaceReclaimed: report.SpaceReclaimed,
+ }, nil
}
-func (s *Service) pruneNetworksInternal(ctx context.Context, options system.PruneNetworksOptions, result *system.PruneAllResult) error {
+func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetworksOptions) (system.PruneAllResult, error) {
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
}
filterArgs := make(client.Filters)
if options.Mode == system.PruneNetworkModeOlderThan {
if strings.TrimSpace(options.Until) == "" {
- return errors.New("network prune mode olderThan requires until")
+ return system.PruneAllResult{}, errors.New("network prune mode olderThan requires until")
}
filterArgs = filterArgs.Add("until", options.Until)
}
report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})
if err != nil {
- return fmt.Errorf("failed to prune networks: %w", err)
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)
}
slog.InfoContext(ctx, "Network prune completed", "networksDeleted", len(report.Report.NetworksDeleted))
- result.NetworksDeleted = report.Report.NetworksDeleted
- return nil
+ return system.PruneAllResult{NetworksDeleted: report.Report.NetworksDeleted}, nil
}
-type pruneResourceInternal struct {
+type pruneResource struct {
Kind string `json:"kind"`
ID string `json:"id"`
Size int64 `json:"size"`
Names []string `json:"names,omitempty"`
}
-type prunePlanInternal struct {
- Request system.PruneAllRequest `json:"request"`
- Resources []pruneResourceInternal `json:"resources"`
+type prunePlan struct {
+ Request system.PruneAllRequest `json:"request"`
+ Resources []pruneResource `json:"resources"`
}
// PruneScheduled freezes exact targets before applying destructive work.
func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req system.PruneAllRequest) (*system.PruneAllResult, bool, error) {
- prune := s.beginSystemPruneInternal(ctx, environmentID, req)
- result := &system.PruneAllResult{Success: true}
- if prune.activityID != "" {
- result.ActivityID = &prune.activityID
- }
+ prune := s.beginSystemPrune(ctx, environmentID, req)
if prune.started.IsAbsent() {
- return result, false, nil
+ return &system.PruneAllResult{
+ Success: true,
+ ActivityID: mo.EmptyableToOption(prune.activityID).ToPointer(),
+ }, false, nil
}
- defer s.finishSystemPruneInternal(environmentID)
+ defer s.finishSystemPrune(environmentID)
fail := func(err error) (*system.PruneAllResult, bool, error) {
- result.Success = false
- result.Errors = append(result.Errors, err.Error())
- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
+ result := &system.PruneAllResult{
+ Success: false,
+ Errors: []string{err.Error()},
+ ActivityID: mo.EmptyableToOption(prune.activityID).ToPointer(),
+ }
+ s.completeSystemPruneActivity(ctx, prune.activityID, result)
return result, true, err
}
- plan, err := s.preparePrunePlanInternal(ctx, req)
+ plan, err := s.preparePrunePlan(ctx, req)
if err != nil {
return fail(err)
}
@@ -549,186 +537,188 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req
return fail(progressErr)
}
previous, _ := jobcontext.Run(ctx)
- err = s.executePrunePlanInternal(ctx, plan, previous, result, false)
- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
+ result, err := s.executePrunePlan(ctx, plan, previous, prune.activityID, false)
+ s.completeSystemPruneActivity(ctx, prune.activityID, result)
if err == nil && result.Success {
err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "scheduled-prune", Status: scheduler.Succeeded, ActivityID: prune.activityID})
}
return result, true, err
}
-func (s *Service) preparePrunePlanInternal(ctx context.Context, req system.PruneAllRequest) (prunePlanInternal, error) {
- plan := prunePlanInternal{Request: req}
+func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return plan, err
+ return prunePlan{}, err
}
if req.Volumes != nil {
s.volumeService.CleanupHelperContainers(ctx)
}
- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: true, Images: true, Volumes: true, BuildCache: req.BuildCache != nil, Verbose: true})
+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{
+ Containers: true,
+ Images: true,
+ Volumes: true,
+ BuildCache: req.BuildCache != nil,
+ Verbose: true,
+ })
if err != nil {
- return plan, err
+ return prunePlan{}, err
}
- if selectPruneContainersErr := s.selectPruneContainersInternal(&plan, usage); selectPruneContainersErr != nil {
- return plan, selectPruneContainersErr
+ containers, err := s.selectPruneContainers(req, usage)
+ if err != nil {
+ return prunePlan{}, err
}
- selectedContainers, imageRefs, volumeRefs := pruneSelectedReferencesInternal(plan.Resources, usage)
- if selectPruneImagesErr := s.selectPruneImagesInternal(&plan, usage, imageRefs); selectPruneImagesErr != nil {
- return plan, selectPruneImagesErr
+ selectedContainers, imageRefs, volumeRefs := pruneSelectedReferences(containers, usage)
+ images, err := s.selectPruneImages(req, usage, imageRefs)
+ if err != nil {
+ return prunePlan{}, err
}
- s.selectPruneVolumesInternal(&plan, usage, volumeRefs)
- if selectPruneNetworksErr := s.selectPruneNetworksInternal(ctx, dockerClient, &plan, selectedContainers); selectPruneNetworksErr != nil {
- return plan, selectPruneNetworksErr
+ networks, err := s.selectPruneNetworks(ctx, dockerClient, req, selectedContainers)
+ if err != nil {
+ return prunePlan{}, err
}
- if selectPruneBuildCacheErr := s.selectPruneBuildCacheInternal(&plan, usage); selectPruneBuildCacheErr != nil {
- return plan, selectPruneBuildCacheErr
+ buildCache, err := s.selectPruneBuildCache(req, usage)
+ if err != nil {
+ return prunePlan{}, err
}
- return plan, nil
+ return prunePlan{
+ Request: req,
+ Resources: slices.Concat(containers, images, s.selectPruneVolumes(req, usage, volumeRefs), networks, buildCache),
+ }, nil
}
-func (s *Service) selectPruneContainersInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {
- req := plan.Request
- var err error
+func (s *Service) selectPruneContainers(req system.PruneAllRequest, usage client.DiskUsageResult) ([]pruneResource, error) {
if req.Containers == nil || req.Containers.Mode == system.PruneContainerModeNone {
- return nil
+ return nil, nil
}
- until := time.Time{}
- if req.Containers.Mode == system.PruneContainerModeOlderThan {
- until, err = s.pruneCutoffInternal(req.Containers.Until)
- if err != nil {
- return err
- }
+ until, err := s.pruneCutoff(req.Containers.Mode == system.PruneContainerModeOlderThan, req.Containers.Until)
+ if err != nil {
+ return nil, err
}
+ var resources []pruneResource
for _, item := range usage.Containers.Items {
if item.State == "running" || item.State == "paused" || item.State == "restarting" || (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {
continue
}
- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "container", ID: item.ID, Size: item.SizeRw})
+ resources = append(resources, pruneResource{
+ Kind: "container",
+ ID: item.ID,
+ Size: item.SizeRw,
+ })
}
- return nil
+ return resources, nil
}
-func (s *Service) selectPruneImagesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, imageRefs map[string]int) error {
- req := plan.Request
- var err error
+func (s *Service) selectPruneImages(req system.PruneAllRequest, usage client.DiskUsageResult, imageRefs map[string]int) ([]pruneResource, error) {
if req.Images == nil || req.Images.Mode == system.PruneImageModeNone {
- return nil
+ return nil, nil
}
- until := time.Time{}
- if req.Images.Mode == system.PruneImageModeOlderThan {
- until, err = s.pruneCutoffInternal(req.Images.Until)
- if err != nil {
- return err
- }
+ until, err := s.pruneCutoff(req.Images.Mode == system.PruneImageModeOlderThan, req.Images.Until)
+ if err != nil {
+ return nil, err
}
+ var resources []pruneResource
for _, item := range usage.Images.Items {
- if item.Containers > int64(
- imageRefs[item.ID],
- ) || (req.Images.Mode == system.PruneImageModeDangling && len(
- item.RepoTags,
- ) > 0) || (!until.IsZero() && !time.Unix(
- item.Created,
- 0,
- ).Before(
- until,
- )) {
+ if item.Containers > int64(imageRefs[item.ID]) ||
+ (req.Images.Mode == system.PruneImageModeDangling && len(item.RepoTags) > 0) ||
+ (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {
continue
}
- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "image", ID: item.ID, Size: item.Size, Names: slices.Clone(item.RepoTags)})
+ resources = append(resources, pruneResource{
+ Kind: "image",
+ ID: item.ID,
+ Size: item.Size,
+ Names: slices.Clone(item.RepoTags),
+ })
}
- return nil
+ return resources, nil
}
-func (s *Service) selectPruneVolumesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, volumeRefs map[string]int) {
- req := plan.Request
- if req.Volumes != nil && req.Volumes.Mode != system.PruneVolumeModeNone {
- for _, item := range usage.Volumes.Items {
- // The backup volume is unused between backups but holds them, so it is never pruned.
- if item.UsageData == nil || item.UsageData.RefCount > int64(volumeRefs[item.Name]) ||
- strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") || s.volumeService.IsBackupVolume(item.Name) {
- continue
- }
- if req.Volumes.Mode == system.PruneVolumeModeAnonymous {
- if _, anonymous := item.Labels["com.docker.volume.anonymous"]; !anonymous {
- continue
- }
- }
- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "volume", ID: item.Name, Size: item.UsageData.Size})
+func (s *Service) selectPruneVolumes(req system.PruneAllRequest, usage client.DiskUsageResult, volumeRefs map[string]int) []pruneResource {
+ if req.Volumes == nil || req.Volumes.Mode == system.PruneVolumeModeNone {
+ return nil
+ }
+ var resources []pruneResource
+ for _, item := range usage.Volumes.Items {
+ // The backup volume is unused between backups but holds them, so it is never pruned.
+ if item.UsageData == nil || item.UsageData.RefCount > int64(volumeRefs[item.Name]) ||
+ strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") || s.volumeService.IsBackupVolume(item.Name) {
+ continue
+ }
+ if _, anonymous := item.Labels["com.docker.volume.anonymous"]; req.Volumes.Mode == system.PruneVolumeModeAnonymous && !anonymous {
+ continue
}
+ resources = append(resources, pruneResource{
+ Kind: "volume",
+ ID: item.Name,
+ Size: item.UsageData.Size,
+ })
}
+ return resources
}
-func (s *Service) selectPruneNetworksInternal(ctx context.Context, dockerClient *client.Client, plan *prunePlanInternal, selectedContainers map[string]bool) error {
- req := plan.Request
- var err error
+func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.Client, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {
if req.Networks == nil || req.Networks.Mode == system.PruneNetworkModeNone {
- return nil
+ return nil, nil
}
- until := time.Time{}
- if req.Networks.Mode == system.PruneNetworkModeOlderThan {
- until, err = s.pruneCutoffInternal(req.Networks.Until)
- if err != nil {
- return err
- }
+ until, err := s.pruneCutoff(req.Networks.Mode == system.PruneNetworkModeOlderThan, req.Networks.Until)
+ if err != nil {
+ return nil, err
}
networks, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
- return err
+ return nil, err
}
+ var resources []pruneResource
+nextNetwork:
for _, item := range networks.Items {
if item.Name == "bridge" || item.Name == "host" || item.Name == "none" || item.Ingress || item.Scope != "local" || (!until.IsZero() && !item.Created.Before(until)) {
continue
}
inspect, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, item.ID, client.NetworkInspectOptions{})
if networkInspectWithCompatibilityErr != nil {
- return networkInspectWithCompatibilityErr
+ return nil, networkInspectWithCompatibilityErr
}
- eligible := true
for id := range inspect.Network.Containers {
if !selectedContainers[id] {
- eligible = false
- break
+ continue nextNetwork
}
}
- if !eligible {
- continue
- }
- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "network", ID: item.ID})
+ resources = append(resources, pruneResource{
+ Kind: "network",
+ ID: item.ID,
+ })
}
- return nil
+ return resources, nil
}
-func (s *Service) selectPruneBuildCacheInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {
- req := plan.Request
- var err error
+func (s *Service) selectPruneBuildCache(req system.PruneAllRequest, usage client.DiskUsageResult) ([]pruneResource, error) {
if req.BuildCache == nil || req.BuildCache.Mode == system.PruneBuildCacheModeNone {
- return nil
+ return nil, nil
}
- until := time.Time{}
- if req.BuildCache.Mode == system.PruneBuildCacheModeOlderThan {
- until, err = s.pruneCutoffInternal(req.BuildCache.Until)
- if err != nil {
- return err
- }
+ until, err := s.pruneCutoff(req.BuildCache.Mode == system.PruneBuildCacheModeOlderThan, req.BuildCache.Until)
+ if err != nil {
+ return nil, err
}
+ var resources []pruneResource
for _, item := range usage.BuildCache.Items {
if item.InUse || (!until.IsZero() && (item.LastUsedAt == nil || !item.LastUsedAt.Before(until))) {
continue
}
- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "build_cache", ID: item.ID, Size: item.Size})
+ resources = append(resources, pruneResource{
+ Kind: "build_cache",
+ ID: item.ID,
+ Size: item.Size,
+ })
}
- return nil
+ return resources, nil
}
-func pruneSelectedReferencesInternal(resources []pruneResourceInternal, usage client.DiskUsageResult) (map[string]bool, map[string]int, map[string]int) {
+func pruneSelectedReferences(containers []pruneResource, usage client.DiskUsageResult) (map[string]bool, map[string]int, map[string]int) {
selectedContainers := make(map[string]bool)
imageRefs := make(map[string]int)
volumeRefs := make(map[string]int)
- for _, resource := range resources {
- if resource.Kind == "container" {
- selectedContainers[resource.ID] = true
- }
+ for _, resource := range containers {
+ selectedContainers[resource.ID] = true
}
for _, item := range usage.Containers.Items {
if !selectedContainers[item.ID] {
@@ -746,17 +736,15 @@ func pruneSelectedReferencesInternal(resources []pruneResourceInternal, usage cl
return selectedContainers, imageRefs, volumeRefs
}
-func (s *Service) pruneCutoffInternal(raw string) (time.Time, error) {
- now := time.Now()
- location := s.location
- if location == nil {
- location = time.UTC
+func (s *Service) pruneCutoff(olderThan bool, raw string) (time.Time, error) {
+ if !olderThan {
+ return time.Time{}, nil
}
if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 {
- return now.Add(-duration), nil
+ return time.Now().Add(-duration), nil
}
for _, format := range []string{time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02"} {
- if value, err := time.ParseInLocation(format, raw, location); err == nil {
+ if value, err := time.ParseInLocation(format, raw, s.location); err == nil {
return value, nil
}
}
@@ -766,133 +754,155 @@ func (s *Service) pruneCutoffInternal(raw string) (time.Time, error) {
return time.Time{}, fmt.Errorf("unsupported prune cutoff %q", raw)
}
-func (s *Service) executePrunePlanInternal(ctx context.Context, plan prunePlanInternal, previous scheduler.Run, result *system.PruneAllResult, recovery bool) error {
+func (s *Service) executePrunePlan(
+ ctx context.Context, plan prunePlan, previous scheduler.Run, activityID string, recovery bool,
+) (*system.PruneAllResult, error) {
+ var containersPruned, imagesDeleted, volumesDeleted, networksDeleted, errs []string
+ var spaceReclaimed, containerSpace, imageSpace, volumeSpace, buildCacheSpace uint64
+ finish := func(err error) (*system.PruneAllResult, error) {
+ if err != nil {
+ errs = append(errs, err.Error())
+ }
+ return &system.PruneAllResult{
+ ContainersPruned: containersPruned,
+ ImagesDeleted: imagesDeleted,
+ VolumesDeleted: volumesDeleted,
+ NetworksDeleted: networksDeleted,
+ SpaceReclaimed: spaceReclaimed,
+ ContainerSpaceReclaimed: containerSpace,
+ ImageSpaceReclaimed: imageSpace,
+ VolumeSpaceReclaimed: volumeSpace,
+ BuildCacheSpaceReclaimed: buildCacheSpace,
+ Success: len(errs) == 0,
+ Errors: errs,
+ ActivityID: mo.EmptyableToOption(activityID).ToPointer(),
+ }, err
+ }
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return err
+ return finish(err)
}
for _, resource := range plan.Resources {
if resource.Kind == "build_cache" {
continue
}
- target := scheduler.TargetOutcome{ResourceType: resource.Kind, ID: "prune:" + resource.Kind + ":" + resource.ID, Status: scheduler.Running}
- if pruneTargetCompletedInternal(previous, target.ID) {
+ targetID := "prune:" + resource.Kind + ":" + resource.ID
+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == targetID && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)
+ }) {
continue
}
- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
- return progressErr
+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: resource.Kind,
+ ID: targetID,
+ Status: scheduler.Running,
+ }); progressErr != nil {
+ return finish(progressErr)
}
- eligible, exists, pruneResourceEligibleErr := s.pruneResourceEligibleInternal(ctx, dockerClient, plan, &resource)
- if pruneResourceEligibleErr != nil {
- return pruneResourceEligibleErr
+ size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, dockerClient, plan, resource)
+ if eligibleErr != nil {
+ return finish(eligibleErr)
}
- switch {
- case !exists:
- target.Status = scheduler.Succeeded
- case !eligible:
- target.Status = scheduler.Skipped
- default:
- pruneResourceEligibleErr = s.removePruneResourceInternal(ctx, dockerClient, resource)
- if pruneResourceEligibleErr != nil && !errdefs.IsNotFound(pruneResourceEligibleErr) {
- target.Status = scheduler.Failed
- target.Message = pruneResourceEligibleErr.Error()
- result.Success = false
- result.Errors = append(result.Errors, pruneResourceEligibleErr.Error())
- } else {
- target.Status = scheduler.Succeeded
- if pruneResourceEligibleErr != nil {
- // Something else removed it first; do not claim its bytes.
- resource.Size = 0
- }
- recordPrunedResourceInternal(resource, result)
+ removeErr := func() error {
+ if !exists || !eligible {
+ return nil
+ }
+ return s.removePruneResource(ctx, dockerClient, resource)
+ }()
+ removeFailed := removeErr != nil && !errdefs.IsNotFound(removeErr)
+ if removeFailed {
+ errs = append(errs, removeErr.Error())
+ }
+ if exists && eligible && !removeFailed {
+ // Something else removed it first; do not claim its bytes.
+ claimed := kit.Ternary(removeErr == nil && size > 0, uint64(size), 0)
+ spaceReclaimed += claimed
+ switch resource.Kind {
+ case "container":
+ containersPruned = append(containersPruned, resource.ID)
+ containerSpace += claimed
+ case "image":
+ imagesDeleted = append(imagesDeleted, resource.ID)
+ imageSpace += claimed
+ case "volume":
+ volumesDeleted = append(volumesDeleted, resource.ID)
+ volumeSpace += claimed
+ case "network":
+ networksDeleted = append(networksDeleted, resource.ID)
}
}
- if targetProgressErr := jobcontext.Progress(ctx, target); targetProgressErr != nil {
- return targetProgressErr
+ if targetProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{
+ ResourceType: resource.Kind,
+ ID: targetID,
+ Status: func() scheduler.RunStatus {
+ switch {
+ case !exists:
+ return scheduler.Succeeded
+ case !eligible:
+ return scheduler.Skipped
+ case removeFailed:
+ return scheduler.Failed
+ default:
+ return scheduler.Succeeded
+ }
+ }(),
+ Message: kit.Ternary(removeFailed, fmt.Sprint(removeErr), ""),
+ }); targetProgressErr != nil {
+ return finish(targetProgressErr)
}
}
- if len(result.ImagesDeleted) > 0 {
- if deleteRecordsForImagesErr := s.imageUpdateService.DeleteRecordsForImages(ctx, result.ImagesDeleted); deleteRecordsForImagesErr != nil {
- return deleteRecordsForImagesErr
+ if len(imagesDeleted) > 0 {
+ if deleteRecordsForImagesErr := s.imageUpdateService.DeleteRecordsForImages(ctx, imagesDeleted); deleteRecordsForImagesErr != nil {
+ return finish(deleteRecordsForImagesErr)
}
}
- return s.executePruneCacheInternal(ctx, dockerClient, plan, result, recovery)
+ reclaimed, err := s.executePruneCache(ctx, dockerClient, plan, recovery)
+ spaceReclaimed += reclaimed
+ buildCacheSpace += reclaimed
+ return finish(err)
}
-func (s *Service) removePruneResourceInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {
- var err error
+func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.Client, resource pruneResource) error {
switch resource.Kind {
case "container":
- _, err = dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})
+ _, err := dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})
+ return err
case "image":
- err = removePruneImageInternal(ctx, dockerClient, resource)
+ for _, name := range resource.Names {
+ inspected, inspectErr := dockerClient.ImageInspect(ctx, name)
+ if errdefs.IsNotFound(inspectErr) {
+ continue
+ }
+ if inspectErr != nil {
+ return inspectErr
+ }
+ if inspected.ID != resource.ID {
+ continue
+ }
+ if _, err := dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false}); err != nil {
+ return err
+ }
+ }
+ _, err := dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})
+ return err
case "volume":
- err = s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)
+ return s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)
case "network":
- err = s.networkService.RemoveNetwork(ctx, resource.ID, user.SystemUser)
+ return s.networkService.RemoveNetwork(ctx, resource.ID, user.SystemUser)
default:
return fmt.Errorf("unknown prune target %q", resource.Kind)
}
- return err
-}
-
-func removePruneImageInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {
- var err error
- for _, name := range resource.Names {
- inspected, inspectErr := dockerClient.ImageInspect(ctx, name)
- if errdefs.IsNotFound(inspectErr) {
- continue
- }
- if inspectErr != nil {
- return inspectErr
- }
- if inspected.ID != resource.ID {
- continue
- }
- _, err = dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false})
- if err != nil {
- return err
- }
- }
- _, err = dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})
- return err
-}
-
-func recordPrunedResourceInternal(resource pruneResourceInternal, result *system.PruneAllResult) {
- switch resource.Kind {
- case "container":
- result.ContainersPruned = append(result.ContainersPruned, resource.ID)
- case "image":
- result.ImagesDeleted = append(result.ImagesDeleted, resource.ID)
- case "volume":
- result.VolumesDeleted = append(result.VolumesDeleted, resource.ID)
- case "network":
- result.NetworksDeleted = append(result.NetworksDeleted, resource.ID)
- }
- if resource.Size <= 0 {
- return
- }
- size := uint64(resource.Size)
- result.SpaceReclaimed += size
- switch resource.Kind {
- case "container":
- result.ContainerSpaceReclaimed += size
- case "image":
- result.ImageSpaceReclaimed += size
- case "volume":
- result.VolumeSpaceReclaimed += size
- }
}
-func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, result *system.PruneAllResult, recovery bool) error {
+func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Client, plan prunePlan, recovery bool) (uint64, error) {
request := plan.Request.BuildCache
if request == nil || request.Mode == system.PruneBuildCacheModeNone {
- return nil
+ return 0, nil
}
if recovery {
usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})
if err != nil {
- return err
+ return 0, err
}
selected := make(map[string]bool)
for _, resource := range plan.Resources {
@@ -902,11 +912,12 @@ func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *c
}
for _, current := range usage.BuildCache.Items {
if selected[current.ID] {
- return errors.New("build cache prune outcome is unconfirmed")
+ return 0, errors.New("build cache prune outcome is unconfirmed")
}
}
- return nil
+ return 0, nil
}
+ var reclaimed uint64
for _, resource := range plan.Resources {
if resource.Kind != "build_cache" {
continue
@@ -921,72 +932,66 @@ func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *c
Filters: filters,
})
if err != nil {
- return err
+ return reclaimed, err
}
- result.SpaceReclaimed += report.Report.SpaceReclaimed
- result.BuildCacheSpaceReclaimed += report.Report.SpaceReclaimed
+ reclaimed += report.Report.SpaceReclaimed
}
- return nil
+ return reclaimed, nil
}
-// pruneResourceEligibleInternal re-checks a planned resource against live state and refreshes its size from that snapshot.
-func (s *Service) pruneResourceEligibleInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, resource *pruneResourceInternal) (eligible, exists bool, err error) {
+// pruneResourceEligible re-checks a planned resource against live state and returns its size from that snapshot.
+func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *client.Client, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {
if resource.Kind == "network" {
inspected, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, resource.ID, client.NetworkInspectOptions{})
if errdefs.IsNotFound(networkInspectWithCompatibilityErr) {
- return false, false, nil
+ return resource.Size, false, false, nil
}
if networkInspectWithCompatibilityErr != nil {
- return false, false, networkInspectWithCompatibilityErr
+ return resource.Size, false, false, networkInspectWithCompatibilityErr
}
- return len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil
+ return resource.Size, len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil
}
- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: resource.Kind == "container", Images: resource.Kind == "image", Volumes: resource.Kind == "volume", Verbose: true})
+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{
+ Containers: resource.Kind == "container",
+ Images: resource.Kind == "image",
+ Volumes: resource.Kind == "volume",
+ Verbose: true,
+ })
if err != nil {
- return false, false, err
+ return resource.Size, false, false, err
}
switch resource.Kind {
case "container":
for _, item := range usage.Containers.Items {
if item.ID == resource.ID {
- resource.Size = item.SizeRw
- return item.State != "running" && item.State != "paused" && item.State != "restarting", true, nil
+ return item.SizeRw, item.State != "running" && item.State != "paused" && item.State != "restarting", true, nil
}
}
case "image":
- return pruneImageEligibleInternal(usage, plan.Request.Images, resource)
- case "volume":
- for _, item := range usage.Volumes.Items {
- if item.Name == resource.ID {
- if item.UsageData == nil {
- return false, true, nil
- }
- resource.Size = item.UsageData.Size
- return item.UsageData.RefCount == 0 && !strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") &&
- !s.volumeService.IsBackupVolume(item.Name), true, nil
+ for _, item := range usage.Images.Items {
+ if item.ID != resource.ID {
+ continue
}
- }
- }
- return false, false, nil
-}
-
-func pruneImageEligibleInternal(usage client.DiskUsageResult, request *system.PruneImagesOptions, resource *pruneResourceInternal) (eligible, exists bool, err error) {
- for _, item := range usage.Images.Items {
- if item.ID == resource.ID {
// Size - SharedSize is the unique layer size Docker counts as reclaimable for an unused image.
- resource.Size = 0
- if item.SharedSize >= 0 {
- resource.Size = item.Size - item.SharedSize
+ reclaimable := kit.Ternary(item.SharedSize >= 0, item.Size-item.SharedSize, 0)
+ if slices.ContainsFunc(item.RepoTags, func(tag string) bool { return !slices.Contains(resource.Names, tag) }) {
+ return reclaimable, false, true, nil
}
- for _, tag := range item.RepoTags {
- if !slices.Contains(resource.Names, tag) {
- return false, true, nil
- }
+ return reclaimable, item.Containers == 0 && (plan.Request.Images.Mode != system.PruneImageModeDangling || len(item.RepoTags) == 0), true, nil
+ }
+ case "volume":
+ for _, item := range usage.Volumes.Items {
+ if item.Name != resource.ID {
+ continue
+ }
+ if item.UsageData == nil {
+ return resource.Size, false, true, nil
}
- return item.Containers == 0 && (request.Mode != system.PruneImageModeDangling || len(item.RepoTags) == 0), true, nil
+ return item.UsageData.Size, item.UsageData.RefCount == 0 && !strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") &&
+ !s.volumeService.IsBackupVolume(item.Name), true, nil
}
}
- return false, false, nil
+ return resource.Size, false, false, nil
}
func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
@@ -998,25 +1003,25 @@ func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous schedule
if target.ID != "prune-plan" || len(target.RecoveryData) == 0 {
continue
}
- var plan prunePlanInternal
+ var plan prunePlan
if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
return outcome, err
}
- prune := s.beginSystemPruneInternal(ctx, previous.EnvironmentID, plan.Request)
+ prune := s.beginSystemPrune(ctx, previous.EnvironmentID, plan.Request)
if prune.started.IsAbsent() {
return outcome, nil
}
- defer s.finishSystemPruneInternal(previous.EnvironmentID) //nolint:gocritic // The matching recovery target always returns before the loop advances.
- result := &system.PruneAllResult{Success: true}
- err := s.executePrunePlanInternal(ctx, plan, previous, result, true)
+ defer s.finishSystemPrune(previous.EnvironmentID) //nolint:gocritic // The matching recovery target always returns before the loop advances.
+ result, err := s.executePrunePlan(ctx, plan, previous, "", true)
+ s.completeSystemPruneActivity(ctx, prune.activityID, result)
if err != nil {
- result.Success = false
- result.Errors = append(result.Errors, err.Error())
- }
- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
- if err != nil {
- outcome.Message = err.Error()
- return outcome, nil
+ return scheduler.Outcome{
+ Status: outcome.Status,
+ Message: err.Error(),
+ ActivityID: outcome.ActivityID,
+ Targets: outcome.Targets,
+ Steps: outcome.Steps,
+ }, nil
}
if !result.Success {
return scheduler.Outcome{Status: scheduler.Partial, Targets: previous.Outcome.Targets}, nil
@@ -1028,12 +1033,3 @@ func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous schedule
}
return outcome, nil
}
-
-func pruneTargetCompletedInternal(previous scheduler.Run, targetID string) bool {
- for _, target := range previous.Outcome.Targets {
- if target.ID == targetID && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped) {
- return true
- }
- }
- return false
-}
diff --git a/backend/internal/system/children/upgrade/handler.go b/backend/internal/system/children/upgrade/handler.go
index b1eaeb365..300d2abb1 100644
--- a/backend/internal/system/children/upgrade/handler.go
+++ b/backend/internal/system/children/upgrade/handler.go
@@ -2,8 +2,8 @@ package upgrade
import (
"context"
- "errors"
"log/slog"
+ "net/http"
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
@@ -97,12 +97,8 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput
err = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body).TargetVersion)
if err != nil {
slog.ErrorContext(ctx, "System upgrade failed", "error", err, "user", user.Username)
-
- if errors.Is(err, common.ErrUpgradeInProgress) {
- return nil, huma.Error409Conflict("Failed to initiate upgrade: " + err.Error())
- }
-
- return nil, huma.Error500InternalServerError("Failed to initiate upgrade: " + err.Error())
+ apiErr := common.ToAPIError(err)
+ return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to initiate upgrade: "+err.Error())
}
message := kit.Ternary(
@@ -124,7 +120,7 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput
// TriggerUpdateAll starts a fleet-wide update: every remote agent in turn, then the manager last.
func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllInput) (*handlerutil.Out[system.EnvironmentUpdateJob], error) {
- if h.cfg != nil && h.cfg.AgentMode {
+ if h.cfg.AgentMode {
return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")
}
@@ -137,8 +133,8 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI
job, err := h.service.StartUpdateAll(ctx, *user)
if err != nil {
- if errors.Is(err, common.ErrUpdateAllInProgress) {
- return nil, huma.Error409Conflict(err.Error())
+ if apiErr := common.ToAPIError(err); apiErr.HTTPStatus() != http.StatusInternalServerError {
+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)
}
return nil, huma.Error500InternalServerError("Failed to initiate upgrade: " + err.Error())
}
@@ -154,7 +150,7 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI
// GetUpdateAllStatus returns the latest update-all job for live progress polling.
func (h *Handler) GetUpdateAllStatus(ctx context.Context, input *UpdateAllStatusInput) (*handlerutil.Out[system.EnvironmentUpdateJob], error) {
- if h.cfg != nil && h.cfg.AgentMode {
+ if h.cfg.AgentMode {
return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")
}
diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.go
index c713f8be3..da000e3ed 100644
--- a/backend/internal/system/children/upgrade/model.go
+++ b/backend/internal/system/children/upgrade/model.go
@@ -71,9 +71,18 @@ type EnvironmentUpdateResult struct {
}
// clearStage drops live-progress fields once the row leaves updating.
-func (r *EnvironmentUpdateResult) clearStage() {
- r.Stage = ""
- r.StageStartedAt = nil
+func (r EnvironmentUpdateResult) clearStage() EnvironmentUpdateResult {
+ return EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: r.EnvironmentID,
+ EnvironmentName: r.EnvironmentName,
+ Status: r.Status,
+ FromVersion: r.FromVersion,
+ FromDigest: r.FromDigest,
+ ToVersion: r.ToVersion,
+ Stage: "",
+ StageStartedAt: nil,
+ Error: r.Error,
+ }
}
// EnvironmentUpdateResults is a JSON-serialized slice of per-environment results,
diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.go
index e243a302e..1c2a0644b 100644
--- a/backend/internal/system/children/upgrade/service.go
+++ b/backend/internal/system/children/upgrade/service.go
@@ -253,30 +253,35 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targ
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- var current container.InspectResponse
- if containerID := strings.TrimSpace(target.ContainerID); containerID != "" {
- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
- if inspectErr != nil {
- // Fall back to a prefix match across every container.
- containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
- if listErr != nil {
- return nil, fmt.Errorf("inspect container: %w", listErr)
- }
- idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })
- if idx < 0 {
- return nil, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))
- }
- if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {
- return nil, fmt.Errorf("inspect container: %w", inspectErr)
+ current, err := func() (container.InspectResponse, error) {
+ containerID := strings.TrimSpace(target.ContainerID)
+ if containerID == "" {
+ self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ if inspectErr != nil {
+ return container.InspectResponse{}, errors.New("get current container: arcane is not running in a Docker container")
}
+ return *self, nil
}
- current = inspect.Container
- } else {
- self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
- if inspectErr != nil {
- return nil, errors.New("get current container: arcane is not running in a Docker container")
+ inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ if inspectErr == nil {
+ return inspect.Container, nil
+ }
+ // Fall back to a prefix match across every container.
+ containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ if listErr != nil {
+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", listErr)
+ }
+ idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })
+ if idx < 0 {
+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))
}
- current = *self
+ if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {
+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", inspectErr)
+ }
+ return inspect.Container, nil
+ }()
+ if err != nil {
+ return nil, err
}
containerName := strings.TrimPrefix(current.Name, "/")
@@ -286,9 +291,11 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targ
targetImage := strings.TrimSpace(target.NewImageRef)
if targetImage == "" {
// A blank target resolves against the version check; a manager-supplied version outranks this instance's own.
- info := kit.FromPtr(s.versionService.GetAppVersionInfo(ctx))
- info.NewestVersion = cmp.Or(strings.TrimSpace(targetVersion), info.NewestVersion)
- if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &info); err != nil {
+ info := s.versionService.GetAppVersionInfo(ctx)
+ if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &versiontypes.Info{
+ NewestVersion: cmp.Or(strings.TrimSpace(targetVersion), kit.FromPtr(info).NewestVersion),
+ NewestDigest: kit.FromPtr(info).NewestDigest,
+ }); err != nil {
return nil, fmt.Errorf("resolve upgrade target image: %w", err)
}
}
@@ -387,18 +394,6 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr
upgradeCmd = append(upgradeCmd, "--pull-image", prepared.pullImage)
}
- config := &container.Config{
- Image: prepared.pullImage,
- Cmd: upgradeCmd,
- // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.
- User: "0:0",
- Env: containerEnv,
- Labels: map[string]string{
- "com.getarcaneapp.arcane.upgrader": "true",
- "com.getarcaneapp.arcane": "true",
- },
- }
-
mounts := append([]mount.Mount{}, runtimeMounts...)
if appDataMount != nil {
mounts = append(mounts, *appDataMount)
@@ -409,42 +404,50 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr
slog.InfoContext(ctx, "Keeping upgrader container after exit (ARCANE_UPGRADE_KEEP_CONTAINER=true)")
}
- hostConfig := &container.HostConfig{
- AutoRemove: !keepUpgraderContainer,
- Mounts: mounts,
- NetworkMode: networkMode,
- }
// Inherit the security context (SELinux label, privileged) that lets Arcane reach the socket on hardened hosts.
- if prepared.current.HostConfig != nil {
- hostConfig.SecurityOpt = slices.Clone(prepared.current.HostConfig.SecurityOpt)
- hostConfig.Privileged = prepared.current.HostConfig.Privileged
- }
+ securityOpt := slices.Clone(kit.FromPtr(prepared.current.HostConfig).SecurityOpt)
// On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.
- hasLabelOpt := slices.ContainsFunc(hostConfig.SecurityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") })
- if !hostConfig.Privileged && !hasLabelOpt {
+ if !kit.FromPtr(prepared.current.HostConfig).Privileged &&
+ !slices.ContainsFunc(securityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") }) {
infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})
if infoErr != nil {
slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)
} else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {
- hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")
- }
- }
-
- var networkingConfig *network.NetworkingConfig
- if networkMode != "" && networkMode.IsUserDefined() {
- networkingConfig = &network.NetworkingConfig{
- EndpointsConfig: map[string]*network.EndpointSettings{
- string(networkMode): {Aliases: []string{"arcane-upgrader"}},
- },
+ securityOpt = append(securityOpt, "label=disable")
}
}
upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())
resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- NetworkingConfig: networkingConfig,
- Name: upgraderName,
+ Config: &container.Config{
+ Image: prepared.pullImage,
+ Cmd: upgradeCmd,
+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.
+ User: "0:0",
+ Env: containerEnv,
+ Labels: map[string]string{
+ "com.getarcaneapp.arcane.upgrader": "true",
+ "com.getarcaneapp.arcane": "true",
+ },
+ },
+ HostConfig: &container.HostConfig{
+ AutoRemove: !keepUpgraderContainer,
+ Mounts: mounts,
+ NetworkMode: networkMode,
+ SecurityOpt: securityOpt,
+ Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,
+ },
+ NetworkingConfig: func() *network.NetworkingConfig {
+ if networkMode == "" || !networkMode.IsUserDefined() {
+ return nil
+ }
+ return &network.NetworkingConfig{
+ EndpointsConfig: map[string]*network.EndpointSettings{
+ string(networkMode): {Aliases: []string{"arcane-upgrader"}},
+ },
+ }
+ }(),
+ Name: upgraderName,
})
if err != nil {
return "", fmt.Errorf("create upgrader container: %w", err)
@@ -578,15 +581,15 @@ func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*En
ManagerVersionAtStart: info.CurrentVersion,
ManagerDigestAtStart: info.CurrentDigest,
ManagerTargetVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),
+ // The manager row stays pending until the final manager step.
+ Results: EnvironmentUpdateResults{{
+ EnvironmentID: environment.LocalEnvironmentID,
+ EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),
+ Status: EnvironmentUpdateResultStatusPending,
+ FromVersion: info.CurrentVersion,
+ ToVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),
+ }},
}
- // The manager row stays pending until the final manager step.
- job.Results = EnvironmentUpdateResults{{
- EnvironmentID: environment.LocalEnvironmentID,
- EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),
- Status: EnvironmentUpdateResultStatusPending,
- FromVersion: info.CurrentVersion,
- ToVersion: job.ManagerTargetVersion,
- }}
// Seed every remote row so the dialog shows the whole fleet at once; the agents step fills any gaps.
if remotes, listErr := s.environments.ListRemoteEnvironments(ctx); listErr != nil {
slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", listErr)
@@ -600,9 +603,11 @@ func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*En
return nil, fmt.Errorf("create update-all job: %w", createUpgradeJobErr)
}
- input := updateAllInput{JobID: job.ID}
- input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)
- instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, input, activitylib.StartRequest{})
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, updateAllInput{
+ JobID: job.ID,
+ KeyID: keyID,
+ }, activitylib.StartRequest{})
if submitErr != nil {
s.markUpdateAllFailed(context.WithoutCancel(ctx), job, fmt.Sprintf("failed to start update-all: %v", submitErr))
return nil, fmt.Errorf("start update-all: %w", submitErr)
@@ -667,8 +672,7 @@ func resolveResumeAction(job *EnvironmentUpdateJob, currentVersion, currentDiges
if now.Sub(job.CreatedAt) > updateAllStaleThreshold {
return resumeAction{markStale: true}
}
- landed := upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)
- return resumeAction{managerSucceeded: landed}
+ return resumeAction{managerSucceeded: upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)}
}
// upgradeLanded reports whether the version or digest moved off its baseline, or already matches target (a version or
@@ -727,12 +731,25 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err
// Only a later stage proves the agent accepted it.
triggered := result.Status == EnvironmentUpdateResultStatusUpdating && result.Stage != "" && result.Stage != EnvironmentUpdateStageChecking
unconfirmed := triggered && result.Stage == EnvironmentUpdateStageStarting
- result.Status = EnvironmentUpdateResultStatusUpdating
+ settle := func(status EnvironmentUpdateResultStatus, message string) {
+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: result.EnvironmentID,
+ EnvironmentName: result.EnvironmentName,
+ Status: status,
+ FromVersion: result.FromVersion,
+ FromDigest: result.FromDigest,
+ ToVersion: result.ToVersion,
+ Stage: result.Stage,
+ StageStartedAt: result.StageStartedAt,
+ Error: message[:min(len(message), updateAllErrorMaxLen)],
+ }
+ }
+ settle(EnvironmentUpdateResultStatusUpdating, result.Error)
if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {
slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)
}
defer func() {
- result.clearStage()
+ *result = result.clearStage()
// Shutdown: leave the row updating at its stage so the redelivered task resumes it.
if err = ctx.Err(); err != nil {
return
@@ -744,15 +761,17 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err
var info versiontypes.Info
if triggered {
- info.CurrentVersion, info.CurrentDigest = result.FromVersion, result.FromDigest
+ info = versiontypes.Info{
+ CurrentVersion: result.FromVersion,
+ CurrentDigest: result.FromDigest,
+ }
} else {
s.setUpdateStage(ctx, job, result, EnvironmentUpdateStageChecking)
versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)
checkErr := s.environments.ProxyJSONRequest(versionCtx, remote.ID, http.MethodGet, "/api/app-version", nil, &info)
cancel()
if checkErr != nil {
- msg := checkErr.Error()
- result.Status, result.Error = updateAllAgentFailureStatus(checkErr), msg[:min(len(msg), updateAllErrorMaxLen)]
+ settle(updateAllAgentFailureStatus(checkErr), checkErr.Error())
return nil, nil
}
// The manager's newest release outranks the agent's own check, so the recorded target tracks what was sent.
@@ -762,16 +781,23 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err
if marshalErr != nil {
slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", remote.ID, "error", marshalErr)
} else {
- info.NewestVersion, triggerBody = newest, body
+ info.NewestVersion, triggerBody = newest, body // keep: versiontypes.Info is decoded by ProxyJSONRequest and has ~20 fields
}
}
- result.FromVersion, result.FromDigest = info.CurrentVersion, info.CurrentDigest
- result.ToVersion = cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest)
-
// The saved starting stage keeps a replay from sending the trigger again, so the trigger waits on it.
- result.Stage, result.StageStartedAt = EnvironmentUpdateStageStarting, new(time.Now())
+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: result.EnvironmentID,
+ EnvironmentName: result.EnvironmentName,
+ Status: result.Status,
+ FromVersion: info.CurrentVersion,
+ FromDigest: info.CurrentDigest,
+ ToVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),
+ Stage: EnvironmentUpdateStageStarting,
+ StageStartedAt: new(time.Now()),
+ Error: result.Error,
+ }
if saveErr := s.db.WithContext(ctx).Save(job).Error; saveErr != nil {
- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error()
+ settle(EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error())
return nil, nil
}
triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)
@@ -781,14 +807,23 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err
triggerErr = resp.RequireSuccess()
}
if triggerErr != nil {
- msg := triggerErr.Error()
- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, msg[:min(len(msg), updateAllErrorMaxLen)]
+ settle(EnvironmentUpdateResultStatusFailed, triggerErr.Error())
return nil, nil
}
// An agent already on the target finds nothing to swap in; confirming would only burn the poll window.
if info.AlreadyOnNewest() {
- result.Status, result.ToVersion = EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion
+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: result.EnvironmentID,
+ EnvironmentName: result.EnvironmentName,
+ Status: EnvironmentUpdateResultStatusUpToDate,
+ FromVersion: result.FromVersion,
+ FromDigest: result.FromDigest,
+ ToVersion: info.CurrentVersion,
+ Stage: result.Stage,
+ StageStartedAt: result.StageStartedAt,
+ Error: result.Error,
+ }
return nil, nil
}
}
@@ -820,15 +855,15 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err
stage(EnvironmentUpdateStageVerifying)
// An unconfirmed request proves nothing by the agent merely matching the target, so it needs a real change.
if upgradeLanded(info.CurrentVersion, info.CurrentDigest, kit.Ternary(unconfirmed, "", result.ToVersion), current.CurrentVersion, current.CurrentDigest) {
- result.Status = EnvironmentUpdateResultStatusUpdated
+ settle(EnvironmentUpdateResultStatusUpdated, result.Error)
return nil, nil
}
}
if unconfirmed {
- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it"
+ settle(EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it")
return nil, nil
}
- result.Status = EnvironmentUpdateResultStatusTriggered
+ settle(EnvironmentUpdateResultStatusTriggered, result.Error)
return nil, nil
}
@@ -843,28 +878,61 @@ func (s *Service) upgradeManagerTask(ctx context.Context, t flow.Task) (any, err
if err != nil {
return nil, err
}
- allowed := false
- if job.Status == EnvironmentUpdateJobStatusRunning {
- if allowed, err = s.authorizeUpdateAll(ctx, job, input.KeyID); err != nil {
- return nil, err
+ allowed, err := func() (bool, error) {
+ if job.Status != EnvironmentUpdateJobStatusRunning {
+ return false, nil
}
+ return s.authorizeUpdateAll(ctx, job, input.KeyID)
+ }()
+ if err != nil {
+ return nil, err
}
if allowed {
// Every agent task has finished, so an agent row still in progress belongs to a task that failed first.
- for i := range job.Results {
- pending := job.Results[i].Status == EnvironmentUpdateResultStatusPending || job.Results[i].Status == EnvironmentUpdateResultStatusUpdating
- if job.Results[i].EnvironmentID != environment.LocalEnvironmentID && pending {
- job.Results[i].clearStage()
- job.Results[i].Status, job.Results[i].Error = EnvironmentUpdateResultStatusFailed, "Agent upgrade did not finish"
+ results := make(EnvironmentUpdateResults, 0, len(job.Results))
+ for _, row := range job.Results {
+ switch {
+ case row.EnvironmentID == environment.LocalEnvironmentID:
+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: row.EnvironmentID,
+ EnvironmentName: row.EnvironmentName,
+ Status: EnvironmentUpdateResultStatusUpdating,
+ FromVersion: row.FromVersion,
+ FromDigest: row.FromDigest,
+ ToVersion: row.ToVersion,
+ Stage: EnvironmentUpdateStageStarting,
+ StageStartedAt: new(time.Now()),
+ Error: row.Error,
+ })
+ case row.Status == EnvironmentUpdateResultStatusPending || row.Status == EnvironmentUpdateResultStatusUpdating:
+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: row.EnvironmentID,
+ EnvironmentName: row.EnvironmentName,
+ Status: EnvironmentUpdateResultStatusFailed,
+ FromVersion: row.FromVersion,
+ FromDigest: row.FromDigest,
+ ToVersion: row.ToVersion,
+ Stage: "",
+ StageStartedAt: nil,
+ Error: "Agent upgrade did not finish",
+ })
+ default:
+ results = append(results, row)
}
}
- manager := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })
- if manager >= 0 {
- job.Results[manager].Status = EnvironmentUpdateResultStatusUpdating
- job.Results[manager].Stage = EnvironmentUpdateStageStarting
- job.Results[manager].StageStartedAt = new(time.Now())
+ manager := slices.IndexFunc(results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })
+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce
+ BaseModel: job.BaseModel,
+ Status: EnvironmentUpdateJobStatusPendingRestart,
+ UserID: job.UserID,
+ Username: job.Username,
+ ManagerVersionAtStart: job.ManagerVersionAtStart,
+ ManagerDigestAtStart: job.ManagerDigestAtStart,
+ ManagerTargetVersion: job.ManagerTargetVersion,
+ Results: results,
+ Error: job.Error,
+ CompletedAt: job.CompletedAt,
}
- job.Status = EnvironmentUpdateJobStatusPendingRestart
// A replay that misses pending_restart would trigger the manager upgrade again, so the trigger waits on it.
if savePendingRestartErr := s.db.WithContext(ctx).Save(job).Error; savePendingRestartErr != nil {
return nil, fmt.Errorf("persist pending_restart before manager upgrade: %w", savePendingRestartErr)
@@ -933,14 +1001,10 @@ func (s *Service) watchManagerUpgrader(ctx context.Context, job *EnvironmentUpda
}
// Let a recreate already underway stop this process; an unobserved run waits out the stale window instead.
- hold := updateAllManagerNoRestartGrace
- if exit == upgraderExitUnobserved {
- hold = time.Until(job.CreatedAt.Add(updateAllStaleThreshold))
- }
select {
case <-ctx.Done():
return
- case <-time.After(hold):
+ case <-time.After(kit.Ternary(exit == upgraderExitUnobserved, time.Until(job.CreatedAt.Add(updateAllStaleThreshold)), updateAllManagerNoRestartGrace)):
}
job, err = s.loadUpdateAllJob(ctx, job.ID)
@@ -973,11 +1037,23 @@ func upsertPendingResult(job *EnvironmentUpdateJob, envID, envName string) int {
if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {
return idx
}
- job.Results = append(job.Results, EnvironmentUpdateResult{
+ pending := EnvironmentUpdateResult{
EnvironmentID: envID,
EnvironmentName: envName,
Status: EnvironmentUpdateResultStatusPending,
- })
+ }
+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce
+ BaseModel: job.BaseModel,
+ Status: job.Status,
+ UserID: job.UserID,
+ Username: job.Username,
+ ManagerVersionAtStart: job.ManagerVersionAtStart,
+ ManagerDigestAtStart: job.ManagerDigestAtStart,
+ ManagerTargetVersion: job.ManagerTargetVersion,
+ Results: append(job.Results, pending),
+ Error: job.Error,
+ CompletedAt: job.CompletedAt,
+ }
return len(job.Results) - 1
}
@@ -998,8 +1074,17 @@ func (s *Service) setUpdateStage(ctx context.Context, job *EnvironmentUpdateJob,
if result.Stage == stage {
return
}
- result.Stage = stage
- result.StageStartedAt = new(time.Now())
+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: result.EnvironmentID,
+ EnvironmentName: result.EnvironmentName,
+ Status: result.Status,
+ FromVersion: result.FromVersion,
+ FromDigest: result.FromDigest,
+ ToVersion: result.ToVersion,
+ Stage: stage,
+ StageStartedAt: new(time.Now()),
+ Error: result.Error,
+ }
if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)
}
@@ -1047,15 +1132,35 @@ func (s *Service) authorizeUpdateAll(ctx context.Context, job *EnvironmentUpdate
}
func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdateJob, reason string) {
- job.Status = EnvironmentUpdateJobStatusFailed
- job.Error = &reason
- job.CompletedAt = new(time.Now())
- for i := range job.Results {
- job.Results[i].clearStage()
- if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {
- job.Results[i].Status = EnvironmentUpdateResultStatusFailed
- job.Results[i].Error = reason
+ results := make(EnvironmentUpdateResults, 0, len(job.Results))
+ for _, row := range job.Results {
+ if row.Status != EnvironmentUpdateResultStatusUpdating {
+ results = append(results, row.clearStage())
+ continue
}
+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: row.EnvironmentID,
+ EnvironmentName: row.EnvironmentName,
+ Status: EnvironmentUpdateResultStatusFailed,
+ FromVersion: row.FromVersion,
+ FromDigest: row.FromDigest,
+ ToVersion: row.ToVersion,
+ Stage: "",
+ StageStartedAt: nil,
+ Error: reason,
+ })
+ }
+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce
+ BaseModel: job.BaseModel,
+ Status: EnvironmentUpdateJobStatusFailed,
+ UserID: job.UserID,
+ Username: job.Username,
+ ManagerVersionAtStart: job.ManagerVersionAtStart,
+ ManagerDigestAtStart: job.ManagerDigestAtStart,
+ ManagerTargetVersion: job.ManagerTargetVersion,
+ Results: results,
+ Error: &reason,
+ CompletedAt: new(time.Now()),
}
if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to mark job failed", "jobId", job.ID, "error", err)
@@ -1084,22 +1189,38 @@ func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdat
// finalizeUpdateAllJob settles the manager row, completes the job and records the audit event.
func (s *Service) finalizeUpdateAllJob(ctx context.Context, job *EnvironmentUpdateJob, managerStatus EnvironmentUpdateResultStatus, managerVersion string) {
- if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID }); idx >= 0 {
- job.Results[idx].Status = managerStatus
- if managerStatus == EnvironmentUpdateResultStatusFailed {
- job.Results[idx].Error = "manager version did not change after upgrade"
- } else {
- job.Results[idx].ToVersion = managerVersion
- }
- }
- job.Status = EnvironmentUpdateJobStatusCompleted
- job.CompletedAt = new(time.Now())
+ results := make(EnvironmentUpdateResults, 0, len(job.Results))
failed := 0
- for i := range job.Results {
- job.Results[i].clearStage()
- if job.Results[i].Status == EnvironmentUpdateResultStatusFailed {
+ for _, row := range job.Results {
+ if row.EnvironmentID == environment.LocalEnvironmentID {
+ row = EnvironmentUpdateResult{ //exhaustruct:enforce
+ EnvironmentID: row.EnvironmentID,
+ EnvironmentName: row.EnvironmentName,
+ Status: managerStatus,
+ FromVersion: row.FromVersion,
+ FromDigest: row.FromDigest,
+ ToVersion: kit.Ternary(managerStatus == EnvironmentUpdateResultStatusFailed, row.ToVersion, managerVersion),
+ Stage: row.Stage,
+ StageStartedAt: row.StageStartedAt,
+ Error: kit.Ternary(managerStatus == EnvironmentUpdateResultStatusFailed, "manager version did not change after upgrade", row.Error),
+ }
+ }
+ if row.Status == EnvironmentUpdateResultStatusFailed {
failed++
}
+ results = append(results, row.clearStage())
+ }
+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce
+ BaseModel: job.BaseModel,
+ Status: EnvironmentUpdateJobStatusCompleted,
+ UserID: job.UserID,
+ Username: job.Username,
+ ManagerVersionAtStart: job.ManagerVersionAtStart,
+ ManagerDigestAtStart: job.ManagerDigestAtStart,
+ ManagerTargetVersion: job.ManagerTargetVersion,
+ Results: results,
+ Error: job.Error,
+ CompletedAt: new(time.Now()),
}
if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)
diff --git a/backend/internal/system/handler.go b/backend/internal/system/handler.go
index 427de5f24..b479d584f 100644
--- a/backend/internal/system/handler.go
+++ b/backend/internal/system/handler.go
@@ -99,9 +99,6 @@ func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoI
}
info := infoResult.Info
- cpuCount := info.NCPU
- memTotal := info.MemTotal
-
// Apply cgroup limits only when running outside Docker (e.g. in LXC).
// In Docker, --cpus/--memory are artificial operator constraints that
// should not cap the host totals shown in the dashboard. The Docker
@@ -111,21 +108,15 @@ func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoI
// dashboard shows what Arcane's host actually has available.
if !cgroup.IsDockerContainer() {
if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {
- if limit := cgroupLimits.MemoryLimit; limit > 0 {
- limitInt := limit
- if memTotal == 0 || limitInt < memTotal {
- memTotal = limitInt
- }
+ if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {
+ info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable
}
- if cgroupLimits.CPUCount > 0 && (cpuCount == 0 || cgroupLimits.CPUCount < cpuCount) {
- cpuCount = cgroupLimits.CPUCount
+ if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {
+ info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable
}
}
}
- info.NCPU = cpuCount
- info.MemTotal = memTotal
-
gitCommit, goVersion, buildTime := extractVersionDetailsFromComponents(version.Components)
return &GetDockerInfoOutput{
@@ -221,17 +212,17 @@ func (h *SystemHandler) ConvertDockerRun(ctx context.Context, input *ConvertDock
return nil, huma.Error500InternalServerError("Failed to convert to Docker Compose format.")
}
- serviceName := ""
- if len(result.Services) > 0 {
- serviceName = result.Services[0].Name
- }
-
return &ConvertDockerRunOutput{
Body: system.ConvertDockerRunResponse{
Success: true,
DockerCompose: string(result.YAML),
EnvVars: strings.TrimSuffix(string(result.EnvFile), "\n"),
- ServiceName: serviceName,
+ ServiceName: func() string {
+ if len(result.Services) == 0 {
+ return ""
+ }
+ return result.Services[0].Name
+ }(),
},
}, nil
}
diff --git a/backend/internal/system/model.go b/backend/internal/system/model.go
index f8635beac..bf913b6c8 100644
--- a/backend/internal/system/model.go
+++ b/backend/internal/system/model.go
@@ -48,16 +48,6 @@ func (b SystemBackupRun) ToDTO() backup.SystemBackupRun {
}
}
-func systemBackupRunFromDTOInternal(dto backup.SystemBackupRun) SystemBackupRun {
- run := SystemBackupRun{
- Size: dto.Size, CreatedAt: dto.CreatedAt, Status: SystemBackupStatus(dto.Status), Trigger: SystemBackupTrigger(dto.Trigger),
- Destination: dto.Destination, LocalSnapshotID: dto.LocalSnapshotID, RemoteSnapshotID: dto.RemoteSnapshotID,
- S3DestinationID: dto.S3DestinationID, S3DestinationName: dto.S3DestinationName, PolicyID: dto.PolicyID, Error: dto.Error,
- }
- run.ID = dto.ID
- return run
-}
-
type SystemBackupPolicy struct {
database.BaseModel
@@ -77,8 +67,3 @@ func (p SystemBackupPolicy) ToDTO() backup.SystemBackupPolicy {
LocalEnabled: p.LocalEnabled, S3Enabled: p.S3Enabled, S3DestinationID: p.S3DestinationID,
}
}
-
-func (p *SystemBackupPolicy) applyDTOInternal(dto backup.SystemBackupPolicy) {
- p.Enabled, p.Schedule, p.RetentionCount = dto.Enabled, dto.Schedule, dto.RetentionCount
- p.LocalEnabled, p.S3Enabled, p.S3DestinationID = dto.LocalEnabled, dto.S3Enabled, dto.S3DestinationID
-}
diff --git a/backend/internal/system/module.go b/backend/internal/system/module.go
index 27df6978c..f3eb46dd9 100644
--- a/backend/internal/system/module.go
+++ b/backend/internal/system/module.go
@@ -3,6 +3,7 @@
package system
import (
+ "cmp"
"net/http"
"github.com/danielgtaylor/huma/v2"
@@ -50,10 +51,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
if m == nil {
m = &Module{}
}
- service := m.service
- if service == nil {
- service = &SystemService{}
- }
+ service := cmp.Or(m.service, &SystemService{})
RegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))
prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))
diff --git a/backend/internal/system/service.go b/backend/internal/system/service.go
index d3519925d..287cc739e 100644
--- a/backend/internal/system/service.go
+++ b/backend/internal/system/service.go
@@ -3,7 +3,6 @@ package system
import (
"cmp"
"context"
- "database/sql"
"errors"
"fmt"
"log/slog"
@@ -25,6 +24,7 @@ import (
"github.com/samber/mo"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
"go.getarcane.app/updater"
"go.getarcane.app/updater/labels"
@@ -91,10 +91,6 @@ func NewSystemService(
actors *francis.Runtime,
cfg *config.Config,
) *SystemService {
- location := time.UTC
- if cfg != nil {
- location = cfg.GetLocation()
- }
s := &SystemService{
db: db,
dockerService: dockerService,
@@ -104,17 +100,13 @@ func NewSystemService(
dockerHostMemoryCache: hot.NewHotCache[struct{}, dockerHostMemoryInfo](hot.LRU, 1).
WithTTL(dockerHostMemoryCacheTTL).
Build(),
- prune: prune.NewService(location, dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),
- upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptionsInternal),
- }
- var sqlDB func() (*sql.DB, error)
- if db != nil {
- sqlDB = db.SQLDB
+ prune: prune.NewService(cfg.GetLocation(), dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),
+ upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptions),
}
s.backup = systembackup.NewService(systembackup.Dependencies{
- Store: s.backupStoreInternal(s3Destinations),
+ Store: s.backupStore(s3Destinations),
DB: db,
- SQLDB: sqlDB,
+ SQLDB: db.SQLDB,
ActorStore: actors.Store,
Docker: dockerService,
Volumes: volumeService,
@@ -124,7 +116,7 @@ func NewSystemService(
Settings: settingsService,
Config: cfg,
RecoveryKeys: recoveryKeys,
- ResolveRuntimeOptions: resolveUpgraderRuntimeOptionsInternal,
+ ResolveRuntimeOptions: resolveUpgraderRuntimeOptions,
})
return s
}
@@ -188,184 +180,130 @@ func (s *SystemService) PruneUpgradeLogs(ctx context.Context, dataDir string, no
return s.upgrade.PruneUpgradeLogs(ctx, dataDir, now)
}
-func (
- s *SystemService,
-) performBatchContainerAction(
- ctx context.Context,
- containers []mobycontainer.Summary,
- actionName string,
- shouldProcess func(
- mobycontainer.Summary,
- ) bool,
- action func(
- context.Context,
- string,
- ) error,
-) *containertypes.ActionResult {
- result := &containertypes.ActionResult{Success: true}
- var mu sync.Mutex
-
- g, groupCtx := errgroup.WithContext(ctx)
- // Limit concurrency to avoid overwhelming Docker daemon
- g.SetLimit(5)
-
- for _, containerSummary := range containers {
- c := containerSummary // capture loop var
- if !shouldProcess(c) {
- continue
- }
-
- g.Go(func() (workerErr error) {
- defer utils.RecoverToError(&workerErr, "system info worker")
-
- err := action(groupCtx, c.ID)
-
- mu.Lock()
- defer mu.Unlock()
-
- if err != nil {
- result.Failed = append(result.Failed, c.ID)
- result.Errors = append(result.Errors, fmt.Sprintf("Failed to %s container %s: %v", actionName, c.ID, err))
- result.Success = false
- } else {
- if actionName == "start" {
- result.Started = append(result.Started, c.ID)
- } else {
- result.Stopped = append(result.Stopped, c.ID)
- }
- }
- return nil
- })
- }
-
- _ = g.Wait()
- return result
-}
-
func (s *SystemService) StartAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {
- return s.startMatchingContainersInternal(ctx, environmentID, startMatchingContainersOptionsInternal{
+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{
+ ActivityType: activitytypes.TypeContainerStart,
+ ActionName: "start",
ResourceName: "All containers",
StartMessage: "Starting all containers",
- FailureMessage: "Starting all containers failed",
SuccessMessage: "Started all containers",
ShouldStart: func(c mobycontainer.Summary) bool { return c.State != "running" },
+ Action: func(ctx context.Context, id string) error {
+ return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)
+ },
})
}
func (s *SystemService) StartAllStoppedContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {
- return s.startMatchingContainersInternal(ctx, environmentID, startMatchingContainersOptionsInternal{
+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{
+ ActivityType: activitytypes.TypeContainerStart,
+ ActionName: "start",
ResourceName: "Stopped containers",
StartMessage: "Starting stopped containers",
- FailureMessage: "Starting stopped containers failed",
SuccessMessage: "Started stopped containers",
ShouldStart: func(c mobycontainer.Summary) bool { return c.State == "exited" },
+ Action: func(ctx context.Context, id string) error {
+ return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)
+ },
+ })
+}
+
+func (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {
+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{
+ ActivityType: activitytypes.TypeContainerStop,
+ ActionName: "stop",
+ ResourceName: "All containers",
+ StartMessage: "Stopping all containers",
+ SuccessMessage: "Stopped all containers",
+ // Skip Arcane container
+ ShouldStart: func(c mobycontainer.Summary) bool { return !labels.IsArcaneContainer(c.Labels) },
+ Action: func(ctx context.Context, id string) error {
+ return s.containerService.StopContainer(ctx, id, usertypes.SystemUser)
+ },
})
}
-type startMatchingContainersOptionsInternal struct {
+type startMatchingContainersOptions struct {
+ ActivityType activitytypes.Type
+ ActionName string
ResourceName string
StartMessage string
- FailureMessage string
SuccessMessage string
ShouldStart func(mobycontainer.Summary) bool
+ Action func(context.Context, string) error
}
-func (s *SystemService) startMatchingContainersInternal(ctx context.Context, environmentID string, opts startMatchingContainersOptionsInternal) (*containertypes.ActionResult, error) {
- activityID := s.startSystemContainerActivityInternal(ctx, environmentID, activitytypes.TypeContainerStart, opts.ResourceName, opts.StartMessage)
- ctx = s.activityService.Track(ctx, activityID)
- containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)
- if err != nil {
- result := &containertypes.ActionResult{
- Success: false,
- Errors: []string{fmt.Sprintf("Failed to list containers: %v", err)},
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- }
- s.completeSystemContainerActivityInternal(ctx, activityID, opts.FailureMessage, result)
- return result, err
- }
-
- result := s.performBatchContainerAction(ctx, containers, "start", opts.ShouldStart, func(ctx context.Context, id string) error {
- return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)
- })
- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.completeSystemContainerActivityInternal(ctx, activityID, opts.SuccessMessage, result)
- return result, nil
-}
-
-func (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {
- activityID := s.startSystemContainerActivityInternal(ctx, environmentID, activitytypes.TypeContainerStop, "All containers", "Stopping all containers")
- ctx = s.activityService.Track(ctx, activityID)
- containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)
- if err != nil {
- result := &containertypes.ActionResult{
- Success: false,
- Errors: []string{fmt.Sprintf("Failed to list containers: %v", err)},
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- }
- s.completeSystemContainerActivityInternal(ctx, activityID, "Stopping all containers failed", result)
- return result, err
- }
-
- result := s.performBatchContainerAction(ctx, containers, "stop",
- func(c mobycontainer.Summary) bool {
- // Skip Arcane container
- return !labels.IsArcaneContainer(c.Labels)
- },
- func(ctx context.Context, id string) error {
- return s.containerService.StopContainer(ctx, id, usertypes.SystemUser)
- })
- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.completeSystemContainerActivityInternal(ctx, activityID, "Stopped all containers", result)
- return result, nil
-}
-
-func (s *SystemService) startSystemContainerActivityInternal(ctx context.Context, environmentID string, activityType activitytypes.Type, resourceName, message string) string {
- if s.activityService == nil {
- return ""
- }
+func (s *SystemService) performBatchContainerAction(ctx context.Context, environmentID string, opts startMatchingContainersOptions) (*containertypes.ActionResult, error) {
localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
EnvironmentID: environmentID,
- Type: activityType,
+ Type: opts.ActivityType,
ResourceType: new("system"),
- ResourceName: &resourceName,
- Step: message,
- LatestMessage: message,
- Metadata: database.JSON{"scope": resourceName},
+ ResourceName: &opts.ResourceName,
+ Step: opts.StartMessage,
+ LatestMessage: opts.StartMessage,
+ Metadata: database.JSON{"scope": opts.ResourceName},
})
if err != nil {
- slog.DebugContext(ctx, "failed to start system container activity", "type", activityType, "error", err)
- return ""
+ slog.DebugContext(ctx, "failed to start system container activity", "type", opts.ActivityType, "error", err)
}
- return localActivity.ID
-}
+ activityID := kit.Ternary(err != nil, "", kit.FromPtr(localActivity).ID)
+ ctx = s.activityService.Track(ctx, activityID)
-func (s *SystemService) completeSystemContainerActivityInternal(ctx context.Context, activityID, successMessage string, result *containertypes.ActionResult) {
- if s.activityService == nil || activityID == "" || result == nil {
- return
+ var mu sync.Mutex
+ var processed, failed, errs []string
+ containers, _, _, _, listErr := s.dockerService.GetAllContainers(ctx)
+ if listErr != nil {
+ errs = append(errs, fmt.Sprintf("Failed to list containers: %v", listErr))
+ } else {
+ g, groupCtx := errgroup.WithContext(ctx)
+ // Limit concurrency to avoid overwhelming Docker daemon
+ g.SetLimit(5)
+ for _, c := range containers {
+ if !opts.ShouldStart(c) {
+ continue
+ }
+ g.Go(func() (workerErr error) {
+ defer utils.RecoverToError(&workerErr, "system info worker")
+ actionErr := opts.Action(groupCtx, c.ID)
+ mu.Lock()
+ defer mu.Unlock()
+ if actionErr != nil {
+ failed = append(failed, c.ID)
+ errs = append(errs, fmt.Sprintf("Failed to %s container %s: %v", opts.ActionName, c.ID, actionErr))
+ return nil
+ }
+ processed = append(processed, c.ID)
+ return nil
+ })
+ }
+ _ = g.Wait()
}
- status := activitytypes.StatusSuccess
- message := successMessage
- var errMessage *string
- if !result.Success || len(result.Errors) > 0 {
- status = activitytypes.StatusFailed
- message = strings.Join(result.Errors, "; ")
- errMessage = &message
+ result := &containertypes.ActionResult{
+ Success: len(errs) == 0,
+ Started: kit.Ternary(opts.ActionName == "start", processed, nil),
+ Stopped: kit.Ternary(opts.ActionName == "start", nil, processed),
+ Failed: failed,
+ Errors: errs,
+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
}
-
- if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {
- slog.DebugContext(ctx, "failed to complete system container activity", "activityId", activityID, "error", err)
+ if activityID != "" {
+ _, completeErr := s.activityService.CompleteActivity(
+ utils.ActivityRuntimeContext(ctx, nil),
+ activityID,
+ kit.Ternary(len(errs) > 0, activitytypes.StatusFailed, activitytypes.StatusSuccess),
+ kit.Ternary(len(errs) > 0, strings.Join(errs, "; "), opts.SuccessMessage),
+ kit.Ternary(len(errs) > 0, new(strings.Join(errs, "; ")), nil),
+ )
+ if completeErr != nil {
+ slog.DebugContext(ctx, "failed to complete system container activity", "activityId", activityID, "error", completeErr)
+ }
}
+ return result, listErr
}
func (s *SystemService) GetDiskUsagePath(ctx context.Context) string {
- cfg := s.settingsService.GetSettingsConfig()
- if cfg == nil {
- return "/"
- }
-
- path := cmp.Or(cfg.DiskUsagePath.Value, "/")
- return path
+ return cmp.Or(s.settingsService.GetSettingsConfig().DiskUsagePath.Value, "/")
}
const dockerHostMemoryCacheTTL = 30 * time.Second
@@ -378,17 +316,16 @@ type dockerHostMemoryInfo struct {
// GetDockerHostMemory reports enclosing guest memory when Docker shares its
// host's cgroup namespace. A false result leaves the caller's baseline intact.
func (s *SystemService) GetDockerHostMemory(ctx context.Context) (uint64, uint64, bool) {
- if runtime.GOOS != "linux" || s.dockerService == nil || s.dockerHostMemoryCache == nil || !cgroup.IsDockerContainer() {
+ if runtime.GOOS != "linux" || !cgroup.IsDockerContainer() {
return 0, 0, false
}
info, found, err := s.dockerHostMemoryCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]dockerHostMemoryInfo, error) {
- metadata, err := s.loadDockerHostMemoryInternal(ctx)
+ metadata, err := s.loadDockerHostMemory(ctx)
if err != nil {
slog.DebugContext(ctx, "Docker host memory accounting unavailable", "error", err)
- // Cache failures too, without retaining expired metadata.
- metadata = dockerHostMemoryInfo{}
}
+ // Cache failures too, without retaining expired metadata.
return map[struct{}]dockerHostMemoryInfo{{}: metadata}, nil
})
if err != nil || !found || info.total == 0 || ctx.Err() != nil {
@@ -402,7 +339,7 @@ func (s *SystemService) GetDockerHostMemory(ctx context.Context) (uint64, uint64
return min(used, info.total), info.total, true
}
-func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (dockerHostMemoryInfo, error) {
+func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMemoryInfo, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
@@ -418,8 +355,7 @@ func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (docke
if err != nil {
return dockerHostMemoryInfo{}, err
}
- if !strings.HasPrefix(inspect.Container.ID, containerID) || inspect.Container.HostConfig == nil ||
- inspect.Container.HostConfig.CgroupnsMode != "host" {
+ if !strings.HasPrefix(inspect.Container.ID, containerID) || kit.FromPtr(inspect.Container.HostConfig).CgroupnsMode != "host" {
return dockerHostMemoryInfo{}, errors.New("current container does not share the Docker host cgroup namespace")
}
// Inspection expands hostname and mountinfo short IDs before cgroup validation.
@@ -438,8 +374,8 @@ func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (docke
return dockerHostMemoryInfo{root: root, total: uint64(info.Info.MemTotal)}, nil
}
-// resolveUpgraderRuntimeOptionsInternal determines how a helper container reaches the Docker daemon.
-func resolveUpgraderRuntimeOptionsInternal(
+// resolveUpgraderRuntimeOptions determines how a helper container reaches the Docker daemon.
+func resolveUpgraderRuntimeOptions(
ctx context.Context,
dockerHost string,
currentContainer *mobycontainer.InspectResponse,
@@ -447,21 +383,18 @@ func resolveUpgraderRuntimeOptionsInternal(
isRunningInDocker func() bool,
selectReachableNetwork func(context.Context, *mobycontainer.InspectResponse, string) string,
) ([]string, []mount.Mount, mobycontainer.NetworkMode, error) {
- containerEnv := vuln.BuildDockerHostEnv(dockerHost)
-
scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)
if err != nil {
return nil, nil, "", fmt.Errorf("resolve docker host %q: %w", dockerHost, err)
}
+ // The upgrader must reach the tcp DOCKER_HOST, so prefer a network the
+ // daemon proxy is actually on over the plain auto heuristic (#3533).
+ if scheme != "unix" && selectReachableNetwork != nil {
+ return vuln.BuildDockerHostEnv(dockerHost), nil, mobycontainer.NetworkMode(selectReachableNetwork(ctx, currentContainer, dockerHost)), nil
+ }
if scheme != "unix" {
- // The upgrader must reach the tcp DOCKER_HOST, so prefer a network the
- // daemon proxy is actually on over the plain auto heuristic (#3533).
- networkMode := docker.SelectAutoNetworkMode(currentContainer)
- if selectReachableNetwork != nil {
- networkMode = selectReachableNetwork(ctx, currentContainer, dockerHost)
- }
- return containerEnv, nil, mobycontainer.NetworkMode(networkMode), nil
+ return vuln.BuildDockerHostEnv(dockerHost), nil, mobycontainer.NetworkMode(docker.SelectAutoNetworkMode(currentContainer)), nil
}
socketSource, err := vuln.ResolveUnixSocketSource(
@@ -474,13 +407,11 @@ func resolveUpgraderRuntimeOptionsInternal(
return nil, nil, "", fmt.Errorf("resolve unix socket source: %w", err)
}
- mounts := []mount.Mount{{
+ return vuln.BuildDockerHostEnv(dockerHost), []mount.Mount{{
Type: mount.TypeBind,
Source: socketSource,
Target: socketPath,
- }}
-
- return containerEnv, mounts, "", nil
+ }}, "", nil
}
const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger, destination, '' AS format, local_snapshot_id, " +
@@ -490,20 +421,68 @@ const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger,
"s3_destination_id, policy_id, error, CASE WHEN policy_id LIKE 'system-volume:%' THEN 'system' ELSE " +
"'volume' END AS type, 'volume' AS resource_type, volume_name AS resource_name FROM volume_backups"
-// backupStoreInternal persists system backup runs and policies for the backup child.
-func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationService) systembackup.Store {
+// backupStore persists system backup runs and policies for the backup child.
+func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) systembackup.Store {
return systembackup.Store{
CreateRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {
- run := systemBackupRunFromDTOInternal(*dto)
+ run := SystemBackupRun{ //exhaustruct:enforce
+ BaseModel: database.BaseModel{
+ ID: dto.ID,
+ CreatedAt: time.Time{},
+ UpdatedAt: nil,
+ },
+ Size: dto.Size,
+ CreatedAt: dto.CreatedAt,
+ Status: SystemBackupStatus(dto.Status),
+ Trigger: SystemBackupTrigger(dto.Trigger),
+ Destination: dto.Destination,
+ LocalSnapshotID: dto.LocalSnapshotID,
+ RemoteSnapshotID: dto.RemoteSnapshotID,
+ S3DestinationID: dto.S3DestinationID,
+ S3DestinationName: dto.S3DestinationName,
+ PolicyID: dto.PolicyID,
+ Error: dto.Error,
+ }
if err := s.db.WithContext(ctx).Create(&run).Error; err != nil {
return err
}
- dto.ID, dto.CreatedAt = run.ID, run.CreatedAt
+ *dto = backuptypes.SystemBackupRun{ //exhaustruct:enforce
+ RemoteAvailable: dto.RemoteAvailable,
+ ActivityID: dto.ActivityID,
+ ID: run.ID,
+ Size: dto.Size,
+ CreatedAt: run.CreatedAt,
+ Status: dto.Status,
+ Trigger: dto.Trigger,
+ Destination: dto.Destination,
+ LocalSnapshotID: dto.LocalSnapshotID,
+ RemoteSnapshotID: dto.RemoteSnapshotID,
+ S3DestinationID: dto.S3DestinationID,
+ S3DestinationName: dto.S3DestinationName,
+ PolicyID: dto.PolicyID,
+ Error: dto.Error,
+ }
return nil
},
SaveRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {
- run := systemBackupRunFromDTOInternal(*dto)
- return s.db.WithContext(ctx).Save(&run).Error
+ return s.db.WithContext(ctx).Save(&SystemBackupRun{ //exhaustruct:enforce
+ BaseModel: database.BaseModel{
+ ID: dto.ID,
+ CreatedAt: time.Time{},
+ UpdatedAt: nil,
+ },
+ Size: dto.Size,
+ CreatedAt: dto.CreatedAt,
+ Status: SystemBackupStatus(dto.Status),
+ Trigger: SystemBackupTrigger(dto.Trigger),
+ Destination: dto.Destination,
+ LocalSnapshotID: dto.LocalSnapshotID,
+ RemoteSnapshotID: dto.RemoteSnapshotID,
+ S3DestinationID: dto.S3DestinationID,
+ S3DestinationName: dto.S3DestinationName,
+ PolicyID: dto.PolicyID,
+ Error: dto.Error,
+ }).Error
},
FailRun: func(ctx context.Context, id, message string) error {
return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", id).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": message}).Error
@@ -578,9 +557,9 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ
return count, err
},
Policies: func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error) {
- policies, err := s.backupPoliciesInternal(ctx)
- if err != nil {
- return nil, err
+ var policies []SystemBackupPolicy
+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {
+ return nil, fmt.Errorf("failed to load system backup policies: %w", err)
}
result := make([]backuptypes.SystemBackupPolicy, len(policies))
for i := range policies {
@@ -609,9 +588,9 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ
unregister func(context.Context, string),
reschedule func(context.Context, *backuptypes.SystemBackupPolicy),
) error {
- existing, err := s.backupPoliciesInternal(ctx)
- if err != nil {
- return err
+ var existing []SystemBackupPolicy
+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&existing).Error; err != nil {
+ return fmt.Errorf("failed to load system backup policies: %w", err)
}
return backup.PolicyReconciliation[SystemBackupPolicy, backuptypes.UpdateSystemBackupPolicy]{
Domain: "system",
@@ -625,7 +604,15 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ
if buildErr := build(ctx, &dto, update); buildErr != nil {
return buildErr
}
- policy.applyDTOInternal(dto)
+ *policy = SystemBackupPolicy{ //exhaustruct:enforce
+ BaseModel: policy.BaseModel,
+ Enabled: dto.Enabled,
+ Schedule: dto.Schedule,
+ RetentionCount: dto.RetentionCount,
+ LocalEnabled: dto.LocalEnabled,
+ S3Enabled: dto.S3Enabled,
+ S3DestinationID: dto.S3DestinationID,
+ }
return nil
},
Unregister: unregister,
@@ -650,21 +637,17 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ
)
}
query = pagination.ApplyFilter(query, "type", params.Filters["type"])
- if params.Sort == "" {
- params.Sort = "createdAt"
- params.Order = pagination.SortDesc
- }
var history []backuptypes.HistoryEntry
- page, err := pagination.PaginateAndSortDB(params, query, &history)
+ page, err := pagination.PaginateAndSortDB(pagination.QueryParams{
+ SearchQuery: params.SearchQuery,
+ SortParams: pagination.SortParams{
+ Sort: cmp.Or(params.Sort, "createdAt"),
+ Order: kit.Ternary(params.Sort == "", pagination.SortDesc, params.Order),
+ },
+ Params: params.Params,
+ Filters: params.Filters,
+ }, query, &history)
return history, page, err
},
}
}
-
-func (s *SystemService) backupPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {
- var policies []SystemBackupPolicy
- if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {
- return nil, fmt.Errorf("failed to load system backup policies: %w", err)
- }
- return policies, nil
-}
diff --git a/backend/internal/system/service_test.go b/backend/internal/system/service_test.go
index bc2799023..da0069e85 100644
--- a/backend/internal/system/service_test.go
+++ b/backend/internal/system/service_test.go
@@ -15,6 +15,7 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
+ "github.com/moby/moby/client"
"github.com/stretchr/testify/require"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
@@ -24,8 +25,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
- systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
@@ -33,7 +35,7 @@ import (
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T) {
+func TestResolveSystemUpgraderRuntimeOptions_TCPDockerHost(t *testing.T) {
currentContainer := &container.InspectResponse{
HostConfig: &container.HostConfig{NetworkMode: "bridge"},
NetworkSettings: &container.NetworkSettings{
@@ -44,7 +46,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)
},
}
- containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptionsInternal(
+ containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptions(
t.Context(),
"tcp://docker-socket-proxy:2375",
currentContainer,
@@ -58,8 +60,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)
require.Equal(t, container.NetworkMode("arcane-test"), networkMode)
}
-func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T) {
- containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptionsInternal(
+func TestResolveSystemUpgraderRuntimeOptions_UnixDockerHost(t *testing.T) {
+ containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptions(
t.Context(),
"unix:///var/run/docker.sock",
nil,
@@ -81,8 +83,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T
}, mounts)
}
-func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testing.T) {
- containerEnv, mounts, _, err := resolveUpgraderRuntimeOptionsInternal(
+func TestResolveSystemUpgraderRuntimeOptions_DefaultDockerHost(t *testing.T) {
+ containerEnv, mounts, _, err := resolveUpgraderRuntimeOptions(
t.Context(),
"",
nil,
@@ -103,8 +105,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testin
}, mounts)
}
-func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionError(t *testing.T) {
- _, _, _, err := resolveUpgraderRuntimeOptionsInternal(
+func TestResolveSystemUpgraderRuntimeOptions_UnixDockerHostResolutionError(t *testing.T) {
+ _, _, _, err := resolveUpgraderRuntimeOptions(
t.Context(),
"unix:///var/run/docker.sock",
nil,
@@ -121,7 +123,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionErr
func TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-volume-history?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupRun{}, &volume.VolumeBackup{}))
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupRun{}, &volume.VolumeBackup{}, &s3.S3Destination{}, &settings.SettingVariable{}))
now := time.Now().UTC()
require.NoError(t, gormDB.Create(&SystemBackupRun{
CreatedAt: now.Add(-2 * time.Minute), Status: SystemBackupStatusSucceeded, Trigger: SystemBackupTriggerManual,
@@ -137,8 +139,14 @@ func TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {
Trigger: volume.VolumeBackupTriggerManual, Destination: volumetypes.BackupDestinationLocal,
Format: volume.VolumeBackupFormatRustic,
}).Error)
- service := &SystemService{db: &database.DB{DB: gormDB}}
- service.backup = systembackup.NewService(systembackup.Dependencies{Store: service.backupStoreInternal(nil)})
+ db := &database.DB{DB: gormDB}
+ settingsService, err := settings.NewSettingsService(t.Context(), db)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })
+ service := NewSystemService(
+ db, nil, nil, nil, nil, nil, settingsService, nil, nil, nil, nil, nil,
+ s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }), nil, nil, &config.Config{},
+ )
systemRows, page, err := service.backup.ListBackupHistory(t.Context(), pagination.QueryParams{
Sort: "createdAt", Order: pagination.SortDesc, Limit: 20,
@@ -162,7 +170,7 @@ func TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {
require.Equal(t, "cache", volumeRows[0].ResourceName)
}
-func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *runs.Admission {
+func newSystemBackupAdmissionGateForTest(t testing.TB) *runs.Admission {
t.Helper()
runtime := francistest.New(t)
gate := runs.NewAdmission(runtime.Service(), t.Name())
@@ -171,26 +179,26 @@ func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *runs.Admission {
return gate
}
-type systemBackupPolicySchedulerInternal struct {
+type systemBackupPolicyScheduler struct {
submitted []scheduler.Request
jobs map[string]scheduler.Job
}
-func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {
+func (s *systemBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {
s.jobs[job.Name()] = job
return nil
}
-func (s *systemBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *systemBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {
delete(s.jobs, name)
}
-func (s *systemBackupPolicySchedulerInternal) HasJob(name string) bool {
+func (s *systemBackupPolicyScheduler) HasJob(name string) bool {
_, ok := s.jobs[name]
return ok
}
-func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
+func (s *systemBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
s.submitted = append(s.submitted, request)
return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil
}
@@ -198,25 +206,51 @@ func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request
func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}, &activity.Activity{}, &activity.ActivityMessage{}))
+ require.NoError(t, gormDB.AutoMigrate(
+ &SystemBackupPolicy{},
+ &SystemBackupRun{},
+ &backup.SystemBackupRecoveryConfig{},
+ &s3.S3Destination{},
+ &activity.Activity{},
+ &activity.ActivityMessage{},
+ &settings.SettingVariable{},
+ &volume.VolumeBackup{},
+ ))
crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})
db := &database.DB{DB: gormDB}
recoveryKeys := backup.NewRecoveryKeyStore(db)
- s3Destinations := s3.NewS3DestinationService(db, nil)
- service := &SystemService{db: db}
- service.backup = systembackup.NewService(systembackup.Dependencies{
- Store: service.backupStoreInternal(s3Destinations),
- DB: db,
- SQLDB: db.SQLDB,
- S3Destinations: s3Destinations,
- Config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
- RecoveryKeys: recoveryKeys,
- })
+ s3Destinations := s3.NewS3DestinationService(db, func(context.Context, string) error { return nil })
+ settingsService, err := settings.NewSettingsService(t.Context(), db)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })
+ gate := newSystemBackupAdmissionGateForTest(t)
+ dockerService := &docker.DockerClientService{Client: func() *client.Client {
+ cli, _ := client.New(client.WithHost("tcp://127.0.0.1:1"), client.WithAPIVersion("1.41"))
+ return cli
+ }()}
+ service := NewSystemService(
+ db,
+ dockerService,
+ nil,
+ nil,
+ volume.NewVolumeService(db, dockerService, nil, nil, nil, nil, nil, nil, nil, nil),
+ nil,
+ settingsService,
+ nil,
+ nil,
+ nil,
+ nil,
+ backup.NewEngine(gate, nil),
+ s3Destinations,
+ recoveryKeys,
+ nil,
+ &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
+ )
harness := flowtest.New(t, activity.NewActivityService(db, nil))
require.NoError(t, service.backup.RegisterWorkflows(harness.Engine))
harness.Start(t)
- jobScheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}
- require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, newSystemBackupAdmissionGateForTestInternal(t)))
+ jobScheduler := &systemBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}
+ require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, gate))
status, err := service.backup.SetRecoveryKey(t.Context(), "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN")
require.NoError(t, err)
@@ -263,12 +297,12 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
job, registered := jobScheduler.jobs[jobName]
require.True(t, registered)
// The scheduled run checks the remote first; a local policy then
- // attempts its backup, which has no engine in this test.
+ // attempts its backup, which has no reachable Docker in this test.
outcome, runErr := job.Run(t.Context())
require.NoError(t, runErr)
if local {
require.Equal(t, scheduler.Failed, outcome.Status)
- require.Contains(t, outcome.Message, "backup engine is unavailable")
+ require.Contains(t, outcome.Message, "failed to create local system recovery snapshot")
} else {
require.Equal(t, scheduler.NeedsAttention, outcome.Status)
require.Equal(t, backup.RemoteDisabledMessage, outcome.Message)
@@ -283,15 +317,13 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}))
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))
db := &database.DB{DB: gormDB}
- service := &SystemService{db: db}
- service.backup = systembackup.NewService(systembackup.Dependencies{
- Store: service.backupStoreInternal(nil),
- DB: db,
- Config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},
- RecoveryKeys: backup.NewRecoveryKeyStore(db),
- })
+ service := NewSystemService(
+ db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+ s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),
+ backup.NewRecoveryKeyStore(db), nil, &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},
+ )
_, err = service.backup.UpdatePolicies(t.Context(), []backuptypes.UpdateSystemBackupPolicy{{
Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 7, LocalEnabled: true,
diff --git a/backend/internal/volume/children/backup/children/policies/service_test.go b/backend/internal/volume/children/backup/children/policies/service_test.go
index 389c88412..dc1f5be65 100644
--- a/backend/internal/volume/children/backup/children/policies/service_test.go
+++ b/backend/internal/volume/children/backup/children/policies/service_test.go
@@ -56,7 +56,11 @@ func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testing
require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))
db := &database.DB{DB: gormDB}
jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}
- service := NewService(Dependencies{DB: db, LatestRun: noLatestRun})
+ service := NewService(Dependencies{
+ DB: db,
+ S3Destinations: s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),
+ LatestRun: noLatestRun,
+ })
gate := newVolumeAdmissionForTest(t)
require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))
@@ -96,7 +100,8 @@ func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicy(t *testing.T) {
}
service := NewService(Dependencies{
- DB: &database.DB{DB: gormDB},
+ DB: &database.DB{DB: gormDB},
+ S3Destinations: s3.NewS3DestinationService(&database.DB{DB: gormDB}, func(context.Context, string) error { return nil }),
LatestRun: func(_ context.Context, policyID string) (*volume.BackupEntry, error) {
return latest[policyID], nil
},