Skip to content

Modern Docker Management, Designed for Everyone

refactor: migrate jobs to use fan out workflows (#4356)

Kyle Mendell1cb757731a43
diff --git a/AGENTS.md b/AGENTS.md
index dead06019..48cb1c5df 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -253,6 +253,8 @@ Determine global admin status through `PermissionSet.IsGlobalAdmin()` or the use
Background jobs implement the scheduler job contract. Wire them through `internal/di` and register them in `internal/bootstrap/jobs_bootstrap.go`.
+Multi-step, fan-out, or resumable jobs use `backend/pkg/scheduler/flow`: Francis workflows that run inside coordinator runs. Follow the migration recipe in its package doc. Steps are idempotent service methods, the last step returns `scheduler.Outcome`, the job registers a `flow.Job`, and `flowtest.AssertDefinitions` guards version bumps. Keep single-operation jobs as ordinary jobs.
+
## Validation
### Tests
diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.go
index c3cc1f943..8378f5c0f 100644
--- a/backend/internal/activity/service.go
+++ b/backend/internal/activity/service.go
@@ -19,6 +19,7 @@ import (
"github.com/samber/mo"
"go.getarcane.app/kit/pkg"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -309,6 +310,18 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
return nil, err
}
+ if req.ID != "" {
+ var existing Activity
+ err := s.db.WithContext(ctx).Where("id = ?", req.ID).Limit(1).Find(&existing).Error
+ if err != nil {
+ return nil, fmt.Errorf("failed to load activity: %w", err)
+ }
+ if existing.ID != "" {
+ dto := activityToDTOInternal(&existing)
+ return &dto, nil
+ }
+ }
+
now := time.Now()
environmentID := cmp.Or(strings.TrimSpace(req.EnvironmentID), "0")
@@ -361,15 +374,30 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
Metadata: cloneJSONInternal(req.Metadata),
CreatedAt: now,
}
+ model.ID = req.ID
if model.Type == "" {
model.Type = activity.TypeAutoUpdate
}
- if err := s.db.WithContext(ctx).Create(model).Error; err != nil {
+ create := s.db.WithContext(ctx)
+ if req.ID != "" {
+ create = create.Clauses(clause.OnConflict{DoNothing: true})
+ }
+ created := create.Create(model)
+ if created.Error != nil || created.RowsAffected == 0 {
if slotRelease != nil {
slotRelease()
}
- return nil, fmt.Errorf("failed to create activity: %w", err)
+ if created.Error != nil {
+ return nil, fmt.Errorf("failed to create activity: %w", created.Error)
+ }
+ // A concurrent start with the same ID created it first.
+ var existing Activity
+ if err := s.db.WithContext(ctx).First(&existing, "id = ?", req.ID).Error; err != nil {
+ return nil, fmt.Errorf("failed to load activity: %w", err)
+ }
+ dto := activityToDTOInternal(&existing)
+ return &dto, nil
}
if slotRelease != nil {
s.registerSlotReleaseInternal(model.ID, slotRelease)
diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.go
index 3b16ffe57..3fa42d4e3 100644
--- a/backend/internal/activity/service_test.go
+++ b/backend/internal/activity/service_test.go
@@ -1064,3 +1064,21 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {
})
}
}
+
+func TestActivityService_StartActivityWithIDIsIdempotent(t *testing.T) {
+ db := setupActivityServiceTestDBInternal(t)
+ service := NewActivityService(db, nil)
+
+ first, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "first"})
+ require.NoError(t, err)
+ require.Equal(t, "workflow-activity", first.ID)
+
+ again, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "redelivered"})
+ require.NoError(t, err)
+ require.Equal(t, first.ID, again.ID)
+ require.Equal(t, "first", again.LatestMessage)
+
+ var count int64
+ require.NoError(t, db.Model(&Activity{}).Where("id = ?", "workflow-activity").Count(&count).Error)
+ require.Equal(t, int64(1), count)
+}
diff --git a/backend/internal/apikey/handler.go b/backend/internal/apikey/handler.go
index 339bab30c..3a0f77292 100644
--- a/backend/internal/apikey/handler.go
+++ b/backend/internal/apikey/handler.go
@@ -148,14 +148,7 @@ func (h *ApiKeyHandler) DeleteApiKey(ctx context.Context, input *DeleteApiKeyInp
return nil, huma.Error500InternalServerError("Failed to delete API key")
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "API key deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("API key deleted successfully", ""), nil
}
// ListMyApiKeys lists API keys owned by the current user (self-service).
@@ -239,12 +232,5 @@ func (h *ApiKeyHandler) DeleteMyApiKey(ctx context.Context, input *DeleteApiKeyI
return nil, huma.Error500InternalServerError("Failed to delete API key")
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "API key deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("API key deleted successfully", ""), nil
}
diff --git a/backend/internal/apns/handler.go b/backend/internal/apns/handler.go
index 2113e7f52..35bab8895 100644
--- a/backend/internal/apns/handler.go
+++ b/backend/internal/apns/handler.go
@@ -105,7 +105,7 @@ func (h *ApnsHandler) DeleteDevice(ctx context.Context, input *DeviceIDInput) (*
if deleteDeviceErr := h.service.DeleteDevice(ctx, user.ID, input.ID); deleteDeviceErr != nil {
return nil, apnsHTTPErrorInternal(deleteDeviceErr)
}
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Device removed"}}}, nil
+ return handlerutil.MessageOutput("Device removed", ""), nil
}
func (h *ApnsHandler) TestDevice(ctx context.Context, input *DeviceIDInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -116,5 +116,5 @@ func (h *ApnsHandler) TestDevice(ctx context.Context, input *DeviceIDInput) (*ha
if testDeviceErr := h.service.TestDevice(ctx, user.ID, input.ID); testDeviceErr != nil {
return nil, apnsHTTPErrorInternal(testDeviceErr)
}
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Test notification queued"}}}, nil
+ return handlerutil.MessageOutput("Test notification queued", ""), nil
}
diff --git a/backend/internal/auth/handler.go b/backend/internal/auth/handler.go
index 66cdb548c..11c8640da 100644
--- a/backend/internal/auth/handler.go
+++ b/backend/internal/auth/handler.go
@@ -25,7 +25,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/labstack/echo/v5"
"github.com/samber/mo"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -272,14 +272,7 @@ func (h *AuthHandler) ChangePassword(ctx context.Context, input *ChangePasswordI
}
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Password changed successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Password changed successfully", ""), nil
}
// LogoutAllOtherSessions revokes every active session for the current user
@@ -295,14 +288,7 @@ func (h *AuthHandler) LogoutAllOtherSessions(ctx context.Context, input *struct{
return nil, huma.Error500InternalServerError("failed to revoke sessions: " + logoutAllOtherSessionsErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "All other sessions signed out",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("All other sessions signed out", ""), nil
}
// validatePreferencesInternal checks the preference values Huma's enum tags
diff --git a/backend/internal/backup/model.go b/backend/internal/backup/model.go
index 6112c21fb..fd6afd886 100644
--- a/backend/internal/backup/model.go
+++ b/backend/internal/backup/model.go
@@ -9,4 +9,5 @@ type SystemBackupRecoveryConfig struct {
EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`
}
+// TableName is the recovery key table.
func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }
diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.go
index 023a35a88..cabd3c168 100644
--- a/backend/internal/backup/service.go
+++ b/backend/internal/backup/service.go
@@ -1,10 +1,9 @@
// Package backup owns the shared Rustic backup engine used by volume and
// system backups: typed repository operations, per-repository serialization
-// and durable run admission.
+// and run admission.
package backup
import (
- "bytes"
"cmp"
"context"
"crypto/rand"
@@ -16,7 +15,6 @@ import (
"fmt"
"io"
"log/slog"
- "maps"
"path"
"regexp"
"slices"
@@ -28,33 +26,33 @@ import (
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
- "github.com/italypaleale/francis/actor"
- "github.com/italypaleale/francis/host/local"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
s3util "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
)
const (
VolumeAdmissionScope = "volume-backup"
SystemAdmissionScope = "system-backup"
- RecoveryKeyConfigID = "system-recovery"
+ // SystemAdmissionID is the one admission key system and system-managed volume backups share.
+ SystemAdmissionID = "system"
+ RecoveryKeyConfigID = "system-recovery"
+ // VolumeRoot is the S3 prefix under which each instance keeps its volume backup repository.
+ VolumeRoot = "arcane-volume-backups"
// rusticRepositoryMissingMessage is how Rustic reports an uninitialized repository.
rusticRepositoryMissingMessage = "No repository config file found"
@@ -71,7 +69,7 @@ type Snapshot struct {
Size int64 `json:"-"`
}
-type rusticSnapshotOutputInternal struct {
+type rusticSnapshotOutput struct {
ID string `json:"id"`
Summary struct {
TotalBytesProcessed int64 `json:"total_bytes_processed"`
@@ -107,7 +105,7 @@ func RootSnapshotInput(source mount.Mount, tags ...string) CreateSnapshotInput {
return CreateSnapshotInput{Mounts: []mount.Mount{source}, Sources: []string{source.Target}, AsPath: "/", Tags: tags}
}
-func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string, error) {
+func snapshotCommand(label string, input CreateSnapshotInput) ([]string, error) {
if len(input.Sources) == 0 {
return nil, errors.New("at least one snapshot source is required")
}
@@ -128,58 +126,40 @@ func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string,
return append(command, input.Sources...), nil
}
-// Engine owns repository serialization and application-owned backup workers.
+// Engine owns repository serialization and backup admission.
type Engine struct {
- imageService *image.ImageService
- admission *runs.Admission
- lifecycleCtx context.Context
- cancel context.CancelFunc
- workers sync.WaitGroup
- stopping bool
- mu sync.Mutex
- repositories map[string]*sync.Mutex
- service *actor.Service
- handlers map[string]func(context.Context, string, []byte, bool) error
- failures map[string]func(context.Context, string, []byte, error) error
- leases map[string]*runs.Lease
- unresolved map[string]backup.DurableRunCommand
- executionReady func() bool
- authorize func(context.Context, backup.DurableRunCommand) error
+ imageService *image.ImageService
+ admission *runs.Admission
+ mu sync.Mutex
+ repositories map[string]*sync.Mutex
+ // held parks admission a manual start won until the run that won it takes it over.
+ held map[scheduler.AdmissionKey]heldLease
+ authorize func(context.Context, backup.Requester) error
}
-func NewEngine(ctx context.Context, admission *runs.Admission, imageService *image.ImageService) *Engine {
- runCtx, cancel := context.WithCancel(ctx)
+// heldLease is a parked lease and the run entitled to take it.
+type heldLease struct {
+ run string
+ lease *runs.Lease
+}
+
+// NewRequester captures the user, and API key if any, behind a manual backup request.
+func NewRequester(ctx context.Context, requestedBy user.Actor, environmentID, permission string) backup.Requester {
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ return backup.Requester{UserID: requestedBy.ID, APIKeyID: keyID, EnvironmentID: environmentID, Permission: permission}
+}
+
+// NewEngine builds the backup engine on the shared admission gate.
+func NewEngine(admission *runs.Admission, imageService *image.ImageService) *Engine {
return &Engine{
- cancel: cancel,
imageService: imageService,
admission: admission,
- lifecycleCtx: runCtx,
- repositories: make(
- map[string]*sync.Mutex,
- ),
- handlers: make(
- map[string]func(
- context.Context,
- string,
- []byte,
- bool,
- ) error,
- ),
- failures: make(
- map[string]func(
- context.Context,
- string,
- []byte,
- error,
- ) error,
- ),
- leases: make(
- map[string]*runs.Lease,
- ),
- unresolved: make(map[string]backup.DurableRunCommand),
+ repositories: make(map[string]*sync.Mutex),
+ held: make(map[scheduler.AdmissionKey]heldLease),
}
}
+// TryAcquireRun takes the admission lease for one backup resource, reporting false while another run holds it.
func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {
if e == nil || e.admission == nil {
return nil, false, errors.New("backup engine is unavailable")
@@ -187,46 +167,79 @@ func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lea
return e.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: scope, ID: id})
}
-func (e *Engine) Stop(ctx context.Context) error {
- if e == nil {
- return nil
- }
+// Hold parks a lease won by a manual start for run until that run's task takes it over with AcquireRun.
+// The returned func releases the lease if the run never took it, such as a start that failed to submit.
+func (e *Engine) Hold(scope, id, run string, lease *runs.Lease) func(context.Context) {
+ key := scheduler.AdmissionKey{Scope: scope, ID: id}
e.mu.Lock()
- e.stopping = true
- e.cancel()
+ e.held[key] = heldLease{run: run, lease: lease}
e.mu.Unlock()
- done := make(chan struct{})
- go func() {
- e.workers.Wait()
+ return func(ctx context.Context) {
e.mu.Lock()
- leases := e.leases
- e.leases = make(map[string]*runs.Lease)
+ untaken := e.held[key].lease == lease
+ if untaken {
+ delete(e.held, key)
+ }
e.mu.Unlock()
- for _, lease := range leases {
+ if untaken {
lease.Release(ctx)
}
- close(done)
- }()
- select {
- case <-done:
+ }
+}
+
+// AcquireRun takes over the lease held for run, or acquires admission otherwise, such as after a restart.
+// A lease parked for another run stays parked, so that run's task still finds its admission.
+func (e *Engine) AcquireRun(ctx context.Context, scope, id, run string) (*runs.Lease, bool, error) {
+ key := scheduler.AdmissionKey{Scope: scope, ID: id}
+ e.mu.Lock()
+ held, ok := e.held[key]
+ if ok = ok && run != "" && held.run == run; ok {
+ delete(e.held, key)
+ }
+ e.mu.Unlock()
+ if ok {
+ return held.lease, true, nil
+ }
+ return e.TryAcquireRun(ctx, scope, id)
+}
+
+// SetAuthorize sets the check Authorize applies to manual backup requesters.
+func (e *Engine) SetAuthorize(authorize func(context.Context, backup.Requester) error) {
+ e.authorize = authorize
+}
+
+// Authorize checks that the requester may still run the backup.
+func (e *Engine) Authorize(ctx context.Context, requester backup.Requester) error {
+ if e.authorize == nil {
return nil
- case <-ctx.Done():
- return ctx.Err()
}
+ return e.authorize(ctx, requester)
+}
+
+// Stop releases leases that no workflow task took over.
+func (e *Engine) Stop(ctx context.Context) error {
+ e.mu.Lock()
+ held := e.held
+ e.held = make(map[scheduler.AdmissionKey]heldLease)
+ e.mu.Unlock()
+ for _, held := range held {
+ held.lease.Release(ctx)
+ }
+ return nil
}
// CreateSnapshot backs the input's sources up into the repository as one
// snapshot. The repository is initialized on first use.
func (e *Engine) CreateSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, label string, input CreateSnapshotInput) (Snapshot, error) {
- command, err := snapshotCommandInternal(label, input)
+ command, err := snapshotCommand(label, input)
if err != nil {
return Snapshot{}, err
}
- output, err := e.runInternal(ctx, dockerClient, repository, password, command, input.Mounts...)
+ output, err := e.run(ctx, dockerClient, repository, password, command, input.Mounts...)
if err != nil {
return Snapshot{}, err
}
- var decoded rusticSnapshotOutputInternal
+ var decoded rusticSnapshotOutput
if unmarshalErr := json.Unmarshal([]byte(output), &decoded); unmarshalErr != nil {
return Snapshot{}, fmt.Errorf("failed to decode Rustic snapshot: %w", unmarshalErr)
}
@@ -242,14 +255,10 @@ func (e *Engine) RestoreSnapshot(ctx context.Context, dockerClient *client.Clien
if options.DeleteExtra {
command = append(command, "--delete")
}
- source := snapshotID + ":/"
- if options.SourcePath != "" {
- source = snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")
- }
- destination := cmp.Or(options.DestinationPath, target.Target)
- command = append(command, "--", source, destination)
+ source := snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")
+ command = append(command, "--", source, cmp.Or(options.DestinationPath, target.Target))
mounts := append([]mount.Mount{target}, options.ExtraMounts...)
- _, err := e.runInternal(ctx, dockerClient, repository, password, command, mounts...)
+ _, err := e.run(ctx, dockerClient, repository, password, command, mounts...)
return err
}
@@ -260,13 +269,14 @@ func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Cli
if recursive {
command = append(command, "--recursive")
}
- cleanedPath := strings.Trim(strings.TrimSpace(filePath), "/")
+ filePath = strings.TrimSpace(filePath)
+ cleanedPath := strings.Trim(filePath, "/")
source := snapshotID + ":/" + cleanedPath
- if cleanedPath != "" && strings.HasSuffix(strings.TrimSpace(filePath), "/") {
+ if cleanedPath != "" && strings.HasSuffix(filePath, "/") {
source += "/"
}
command = append(command, "--", source)
- output, err := e.runInternal(ctx, dockerClient, repository, password, command)
+ output, err := e.run(ctx, dockerClient, repository, password, command)
if err != nil {
return nil, fmt.Errorf("failed to list Rustic snapshot: %w", err)
}
@@ -278,21 +288,21 @@ func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Cli
// Rustic's JSON listing omits node types; the stable long listing restores them.
longCommand := slices.Clone(command)
longCommand[1] = "--long"
- longOutput, runErr := e.runInternal(ctx, dockerClient, repository, password, longCommand)
+ longOutput, runErr := e.run(ctx, dockerClient, repository, password, longCommand)
if runErr != nil {
return nil, fmt.Errorf("failed to list Rustic snapshot metadata: %w", runErr)
}
- files, runErr = markSnapshotDirectoriesInternal(files, longOutput)
+ files, runErr = markSnapshotDirectories(files, longOutput)
if runErr != nil {
return nil, runErr
}
}
- return qualifySnapshotListingInternal(files, cleanedPath), nil
+ return qualifySnapshotListing(files, cleanedPath), nil
}
-func qualifySnapshotListingInternal(files []string, snapshotPath string) []string {
+// qualifySnapshotListing prefixes listed files with the cleaned snapshot path they were listed under.
+func qualifySnapshotListing(files []string, prefix string) []string {
qualified := slices.Clone(files)
- prefix := strings.Trim(strings.TrimSpace(snapshotPath), "/")
if prefix == "" {
return qualified
}
@@ -307,7 +317,7 @@ func qualifySnapshotListingInternal(files []string, snapshotPath string) []strin
return qualified
}
-func markSnapshotDirectoriesInternal(files []string, longOutput string) ([]string, error) {
+func markSnapshotDirectories(files []string, longOutput string) ([]string, error) {
if len(files) == 0 {
if strings.TrimSpace(longOutput) != "" {
return nil, errors.New("rustic file and metadata listings have different lengths")
@@ -332,7 +342,7 @@ func markSnapshotDirectoriesInternal(files []string, longOutput string) ([]strin
// ReadSnapshotTextFile returns one text file from a snapshot.
func (e *Engine) ReadSnapshotTextFile(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID, filePath string) (string, error) {
- output, err := e.runInternal(ctx, dockerClient, repository, password, []string{
+ output, err := e.run(ctx, dockerClient, repository, password, []string{
"dump", "--archive", "content", "--", snapshotID + ":/" + strings.TrimPrefix(filePath, "/"),
})
if err != nil {
@@ -357,11 +367,11 @@ func RunSnapshotTag(runID string) string { return "arcane-run:" + runID }
// ConfirmRunSnapshot makes a validated snapshot discoverable by run ID.
func (e *Engine) ConfirmRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, error) {
- if !fullSnapshotIDInternal(snapshotID) {
+ if !fullSnapshotID(snapshotID) {
return Snapshot{}, errors.New("a full snapshot ID is required")
}
command := []string{"tag", "--add", RunSnapshotTag(runID), "--", snapshotID}
- if _, err := e.runInternal(ctx, dockerClient, repository, password, command); err != nil {
+ if _, err := e.run(ctx, dockerClient, repository, password, command); err != nil {
return Snapshot{}, err
}
// Tagging rewrites the snapshot and changes its ID.
@@ -403,14 +413,14 @@ func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Clien
// ListSnapshots enumerates every snapshot in the repository.
func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {
- output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
+ output, err := e.run(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
if err != nil {
return nil, err
}
- return decodeSnapshotsInternal(output)
+ return decodeSnapshots(output)
}
-func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {
+func decodeSnapshots(output string) ([]DiscoveredSnapshot, error) {
trimmedOutput := strings.TrimSpace(output)
if trimmedOutput == "" || trimmedOutput[0] != '[' {
return nil, errors.New("invalid Rustic snapshot listing: expected an array")
@@ -439,25 +449,34 @@ func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {
return nil, errors.New("invalid Rustic snapshot listing: mixed flat and grouped entries")
}
grouped = hasSnapshots
- if !hasSnapshots {
- snapshot, err := decodeSnapshotInternal(entry)
- if err != nil {
- return nil, fmt.Errorf("invalid Rustic snapshot listing entry %d: %w", index, err)
+ members := []jsontext.Value{entry}
+ if hasSnapshots {
+ if groupEntries.Kind() != '[' {
+ return nil, fmt.Errorf("invalid Rustic snapshot group %d: expected a snapshots array", index)
+ }
+ members = nil
+ if err := json.Unmarshal(groupEntries, &members); err != nil {
+ return nil, fmt.Errorf("failed to decode Rustic snapshot group %d: %w", index, err)
}
- snapshots = append(snapshots, snapshot)
- continue
- }
- if groupEntries.Kind() != '[' {
- return nil, fmt.Errorf("invalid Rustic snapshot group %d: expected a snapshots array", index)
- }
- var members []jsontext.Value
- if err := json.Unmarshal(groupEntries, &members); err != nil {
- return nil, fmt.Errorf("failed to decode Rustic snapshot group %d: %w", index, err)
}
for memberIndex, member := range members {
- snapshot, err := decodeSnapshotInternal(member)
- if err != nil {
- return nil, fmt.Errorf("invalid Rustic snapshot group %d entry %d: %w", index, memberIndex, err)
+ location := kit.Ternary(hasSnapshots, fmt.Sprintf("group %d entry %d", index, memberIndex), fmt.Sprintf("listing entry %d", index))
+ var memberFields map[string]jsontext.Value
+ if member.Kind() != '{' {
+ return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a snapshot object", location)
+ }
+ if err := json.Unmarshal(member, &memberFields); err != nil {
+ return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot fields: %w", location, err)
+ }
+ if _, nested := memberFields["snapshots"]; nested {
+ return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a flat snapshot object", location)
+ }
+ var snapshot DiscoveredSnapshot
+ if err := json.Unmarshal(member, &snapshot); err != nil {
+ return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot payload: %w", location, err)
+ }
+ if !fullSnapshotID(snapshot.ID) {
+ return nil, fmt.Errorf("invalid Rustic snapshot %s: invalid full snapshot ID", location)
}
snapshots = append(snapshots, snapshot)
}
@@ -465,28 +484,7 @@ func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {
return snapshots, nil
}
-func decodeSnapshotInternal(raw jsontext.Value) (DiscoveredSnapshot, error) {
- if raw.Kind() != '{' {
- return DiscoveredSnapshot{}, errors.New("expected a snapshot object")
- }
- var fields map[string]jsontext.Value
- if err := json.Unmarshal(raw, &fields); err != nil {
- return DiscoveredSnapshot{}, fmt.Errorf("failed to decode Rustic snapshot fields: %w", err)
- }
- if _, grouped := fields["snapshots"]; grouped {
- return DiscoveredSnapshot{}, errors.New("expected a flat snapshot object")
- }
- var snapshot DiscoveredSnapshot
- if err := json.Unmarshal(raw, &snapshot); err != nil {
- return DiscoveredSnapshot{}, fmt.Errorf("failed to decode Rustic snapshot payload: %w", err)
- }
- if !fullSnapshotIDInternal(snapshot.ID) {
- return DiscoveredSnapshot{}, errors.New("invalid full snapshot ID")
- }
- return snapshot, nil
-}
-
-func fullSnapshotIDInternal(id string) bool {
+func fullSnapshotID(id string) bool {
if len(id) != 64 {
return false
}
@@ -499,30 +497,25 @@ func fullSnapshotIDInternal(id string) bool {
func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string, snapshotIDs []string) error {
requested := make([]string, 0, len(snapshotIDs))
for _, id := range snapshotIDs {
- if !fullSnapshotIDInternal(id) {
+ if !fullSnapshotID(id) {
return errors.New("a full snapshot ID is required")
}
requested = append(requested, strings.ToLower(id))
}
requested = kit.Unique(requested)
- if e == nil {
- return errors.New("backup engine is unavailable")
- }
- if strings.TrimSpace(repository.ID) == "" {
- return errors.New("backup repository ID is required")
- }
- defer e.lockRepositoryInternal(repository.ID)()
- if err := ctx.Err(); err != nil {
+ unlock, err := e.lockRepository(ctx, repository.ID)
+ if err != nil {
return err
}
- output, err := e.runContainerInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
+ defer unlock()
+ output, err := e.runContainer(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
if len(snapshotIDs) == 0 && err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
return nil
}
if err != nil {
return err
}
- snapshots, err := decodeSnapshotsInternal(output)
+ snapshots, err := decodeSnapshots(output)
if err != nil {
return err
}
@@ -542,7 +535,7 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien
if len(command) == 2 {
command = []string{"prune"}
}
- _, err = e.runContainerInternal(ctx, dockerClient, repository, password, command)
+ _, err = e.runContainer(ctx, dockerClient, repository, password, command)
return err
}
@@ -551,16 +544,12 @@ func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *cli
if strings.TrimSpace(newPassword) == "" {
return errors.New("new repository password is required")
}
- _, err := e.runInternal(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})
+ _, err := e.run(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})
return err
}
-// Replicate copies one snapshot between repositories by restoring it into a
-// temporary volume and backing that volume up into the target repository. The
-// source is read once from the repository, never from the live data. Rustic's
-// native `copy` would move only missing packs, but it addresses the target via
-// a TOML config profile, which the env-only Repository cannot express yet —
-// the materialize-and-rebackup here trades disk and I/O for that simplicity.
+// Replicate copies one snapshot into another repository through a temporary volume, never the live data.
+// Rustic's native `copy` needs a TOML profile for the target, which the env-only Repository cannot express.
func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {
temporaryVolume := "arcane-rustic-copy-" + uuid.New().String()
if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {
@@ -581,58 +570,47 @@ func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, fro
return snapshot, nil
}
-func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
- if e == nil {
- return "", errors.New("backup engine is unavailable")
- }
- if strings.TrimSpace(repository.ID) == "" {
- return "", errors.New("backup repository ID is required")
- }
- defer e.lockRepositoryInternal(repository.ID)()
- if err := ctx.Err(); err != nil {
+func (e *Engine) run(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
+ unlock, err := e.lockRepository(ctx, repository.ID)
+ if err != nil {
return "", err
}
- return e.runContainerInternal(ctx, dockerClient, repository, password, command, extraMounts...)
+ defer unlock()
+ return e.runContainer(ctx, dockerClient, repository, password, command, extraMounts...)
}
-func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {
- if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err == nil {
- return nil
- }
- if e.imageService == nil {
- return errors.New("image service is unavailable")
- }
- if err := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); err != nil {
- return fmt.Errorf("failed to pull official Rustic image: %w", err)
+// runContainer runs one Rustic command; callers hold the repository lock.
+func (e *Engine) runContainer(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
+ if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err != nil {
+ if e.imageService == nil {
+ return "", errors.New("image service is unavailable")
+ }
+ if pullErr := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); pullErr != nil {
+ return "", fmt.Errorf("failed to pull official Rustic image: %w", pullErr)
+ }
}
- return nil
-}
-
-func arcaneNetworkModeInternal(ctx context.Context, dockerClient *client.Client) container.NetworkMode {
+ var networkMode container.NetworkMode
arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
if err != nil || arcane == nil || arcane.ID == "" {
slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)
- return ""
- }
- return container.NetworkMode("container:" + arcane.ID)
-}
-
-func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
- if err := e.ensureImageInternal(ctx, dockerClient); err != nil {
- return "", err
+ } else {
+ networkMode = container.NetworkMode("container:" + arcane.ID)
}
- mounts := append([]mount.Mount{}, repository.Mounts...)
- mounts = append(mounts, extraMounts...)
- return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))
+ mounts := append(slices.Clone(repository.Mounts), extraMounts...)
+ return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, networkMode)
}
var (
ErrRecoveryKeyNotConfigured = errors.New("recovery key is not configured")
+ // ErrBackupSettled reports a checkpointed backup whose own attempt already failed it, so there is nothing to
+ // resume; a retry starts a new backup instead.
+ ErrBackupSettled = errors.New("the checkpointed backup already failed")
// RecoveryKeyFormat is 8 hyphenated groups of 6 base32 characters, used verbatim as the Rustic password.
RecoveryKeyFormat = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)
)
+// ValidateRecoveryKey checks that a key has the generated recovery key format.
func ValidateRecoveryKey(key string) error {
if !RecoveryKeyFormat.MatchString(strings.TrimSpace(key)) {
return errors.New("enter the generated recovery key (8 groups of 6 characters)")
@@ -640,6 +618,7 @@ func ValidateRecoveryKey(key string) error {
return nil
}
+// GenerateRecoveryKey creates a random recovery key in RecoveryKeyFormat.
func GenerateRecoveryKey() (string, error) {
raw := make([]byte, 30)
if _, err := rand.Read(raw); err != nil {
@@ -658,10 +637,12 @@ type RecoveryKeyStore struct {
db *database.DB
}
+// NewRecoveryKeyStore builds the recovery key store.
func NewRecoveryKeyStore(db *database.DB) *RecoveryKeyStore {
return &RecoveryKeyStore{db: db}
}
+// Configured reports whether a recovery key is stored.
func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {
var count int64
if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).
@@ -673,10 +654,40 @@ func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {
// Get returns the decrypted recovery key or ErrRecoveryKeyNotConfigured.
func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {
+ return s.load(ctx, RecoveryKeyConfigID)
+}
+
+// Set validates and stores the recovery key, encrypted.
+func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
+ return s.save(ctx, RecoveryKeyConfigID, recoveryKey)
+}
+
+// HoldRunKey keeps a recovery key entered for one backup run, encrypted, until ReleaseRunKey, so the run can
+// resume after a restart without it ever entering the run's workflow payload.
+func (s *RecoveryKeyStore) HoldRunKey(ctx context.Context, runID, recoveryKey string) error {
+ return s.save(ctx, "run:"+runID, recoveryKey)
+}
+
+// RunKey returns the key held for a run, or "" when none is.
+func (s *RecoveryKeyStore) RunKey(ctx context.Context, runID string) (string, error) {
+ key, err := s.load(ctx, "run:"+runID)
+ if errors.Is(err, ErrRecoveryKeyNotConfigured) {
+ return "", nil
+ }
+ return key, err
+}
+
+// ReleaseRunKey forgets the key held for a run.
+func (s *RecoveryKeyStore) ReleaseRunKey(ctx context.Context, runID string) error {
+ if err := s.db.WithContext(ctx).Where("id = ?", "run:"+runID).Delete(&SystemBackupRecoveryConfig{}).Error; err != nil {
+ return fmt.Errorf("failed to release backup run recovery key: %w", err)
+ }
+ return nil
+}
+
+func (s *RecoveryKeyStore) load(ctx context.Context, id string) (string, error) {
var config SystemBackupRecoveryConfig
- if err := s.db.WithContext(ctx).
- Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).
- First(&config).Error; err != nil {
+ if err := s.db.WithContext(ctx).Where("id = ? AND encrypted_recovery_key <> ''", id).First(&config).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return "", ErrRecoveryKeyNotConfigured
}
@@ -689,7 +700,7 @@ func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {
return key, nil
}
-func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
+func (s *RecoveryKeyStore) save(ctx context.Context, id, recoveryKey string) error {
if err := ValidateRecoveryKey(recoveryKey); err != nil {
return err
}
@@ -697,14 +708,40 @@ func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
if err != nil {
return fmt.Errorf("failed to encrypt recovery key: %w", err)
}
- config := SystemBackupRecoveryConfig{ID: RecoveryKeyConfigID, EncryptedRecoveryKey: encrypted}
- if saveRecoveryKeyErr := s.db.WithContext(ctx).Save(&config).Error; saveRecoveryKeyErr != nil {
- return fmt.Errorf("failed to save recovery key: %w", saveRecoveryKeyErr)
+ if saveErr := s.db.WithContext(ctx).Save(&SystemBackupRecoveryConfig{BaseModel: database.BaseModel{ID: id}, EncryptedRecoveryKey: encrypted}).Error; saveErr != nil {
+ return fmt.Errorf("failed to save recovery key: %w", saveErr)
}
return nil
}
-func (e *Engine) lockRepositoryInternal(id string) func() {
+// AcceptedTarget is the progress a manual backup's workflow starts with, so startup recovery protects
+// its record from the moment the backup is accepted.
+func AcceptedTarget(backupID string) scheduler.TargetOutcome {
+ data, _ := json.Marshal(struct {
+ BackupID string `json:"backupId"`
+ }{backupID})
+ return scheduler.TargetOutcome{ResourceType: "backup", ID: "accepted:" + backupID, Status: scheduler.Queued, RecoveryData: data}
+}
+
+// AcceptedBackupID returns the backup a manual backup workflow's run owns, from its AcceptedTarget.
+func AcceptedBackupID(run scheduler.Run) string {
+ index := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return strings.HasPrefix(target.ID, "accepted:")
+ })
+ if index < 0 {
+ return ""
+ }
+ return strings.TrimPrefix(run.Outcome.Targets[index].ID, "accepted:")
+}
+
+// lockRepository serializes commands against one repository and returns its unlock func.
+func (e *Engine) lockRepository(ctx context.Context, id string) (func(), error) {
+ if e == nil {
+ return nil, errors.New("backup engine is unavailable")
+ }
+ if strings.TrimSpace(id) == "" {
+ return nil, errors.New("backup repository ID is required")
+ }
e.mu.Lock()
lock := e.repositories[id]
if lock == nil {
@@ -713,7 +750,11 @@ func (e *Engine) lockRepositoryInternal(id string) func() {
}
e.mu.Unlock()
lock.Lock()
- return lock.Unlock
+ if err := ctx.Err(); err != nil {
+ lock.Unlock()
+ return nil, err
+ }
+ return lock.Unlock, nil
}
// PolicyReconciliation reconciles a full set of policy updates against the
@@ -737,47 +778,70 @@ type PolicyReconciliation[P, U any] struct {
Reschedule func(ctx context.Context, policy *P)
}
+// Run applies updates as the full policy set: it saves them, removes the rest, and reschedules their jobs.
func (r PolicyReconciliation[P, U]) Run(ctx context.Context, updates []U) error {
- policies, kept, err := r.buildInternal(ctx, updates)
- if err != nil {
- return err
- }
- if persistErr := r.persistInternal(ctx, policies, kept); persistErr != nil {
- return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, persistErr)
- }
+ byID := make(map[string]P, len(r.Existing))
for i := range r.Existing {
- if _, ok := kept[r.ID(&r.Existing[i])]; !ok {
- r.Unregister(ctx, r.ID(&r.Existing[i]))
- }
+ byID[r.ID(&r.Existing[i])] = r.Existing[i]
}
- for i := range policies {
- r.Reschedule(ctx, &policies[i])
+ policies := make([]P, 0, len(updates))
+ kept := make(map[string]struct{}, len(updates))
+ for _, update := range updates {
+ policy := r.New()
+ if updateID := r.UpdateID(update); updateID != "" {
+ var ok bool
+ policy, ok = byID[updateID]
+ if !ok {
+ return fmt.Errorf("%s backup policy not found", r.Domain)
+ }
+ if _, duplicate := kept[updateID]; duplicate {
+ return fmt.Errorf("duplicate %s backup policy", r.Domain)
+ }
+ }
+ if err := r.Build(ctx, &policy, update); err != nil {
+ return err
+ }
+ if policyID := r.ID(&policy); policyID != "" {
+ kept[policyID] = struct{}{}
+ }
+ policies = append(policies, policy)
}
- return nil
-}
+ removed := slices.DeleteFunc(slices.Clone(r.Existing), func(policy P) bool {
+ _, ok := kept[r.ID(&policy)]
+ return ok
+ })
-func (r PolicyReconciliation[P, U]) persistInternal(ctx context.Context, policies []P, kept map[string]struct{}) error {
- if r.Persist != nil {
- return r.Persist(ctx, policies)
- }
- if r.DB == nil {
- return errors.New("backup policy database is unavailable")
- }
- return r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for i := range policies {
- if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {
- return saveErr
+ var persistErr error
+ switch {
+ case r.Persist != nil:
+ persistErr = r.Persist(ctx, policies)
+ case r.DB == nil:
+ persistErr = errors.New("backup policy database is unavailable")
+ default:
+ persistErr = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for i := range policies {
+ if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {
+ return saveErr
+ }
}
- }
- for i := range r.Existing {
- if _, ok := kept[r.ID(&r.Existing[i])]; !ok {
- if deleteErr := tx.Delete(&r.Existing[i]).Error; deleteErr != nil {
+ for i := range removed {
+ if deleteErr := tx.Delete(&removed[i]).Error; deleteErr != nil {
return deleteErr
}
}
- }
- return nil
- })
+ return nil
+ })
+ }
+ if persistErr != nil {
+ return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, persistErr)
+ }
+ for i := range removed {
+ r.Unregister(ctx, r.ID(&removed[i]))
+ }
+ for i := range policies {
+ r.Reschedule(ctx, &policies[i])
+ }
+ return nil
}
// ValidatePolicyUpdate applies the shared cron, retention, and destination rules used by backup policies.
@@ -809,53 +873,20 @@ func ValidatePolicyUpdate(ctx context.Context, domain string, update backup.Upda
return update, nil
}
-// buildInternal maps the updates onto existing or new policy rows and reports
-// which existing IDs survive the reconciliation.
-func (r PolicyReconciliation[P, U]) buildInternal(ctx context.Context, updates []U) ([]P, map[string]struct{}, error) {
- byID := make(map[string]P, len(r.Existing))
- for i := range r.Existing {
- byID[r.ID(&r.Existing[i])] = r.Existing[i]
- }
- policies := make([]P, 0, len(updates))
- kept := make(map[string]struct{}, len(updates))
- for _, update := range updates {
- policy := r.New()
- updateID := r.UpdateID(update)
- if updateID != "" {
- var ok bool
- policy, ok = byID[updateID]
- if !ok {
- return nil, nil, fmt.Errorf("%s backup policy not found", r.Domain)
- }
- if _, duplicate := kept[updateID]; duplicate {
- return nil, nil, fmt.Errorf("duplicate %s backup policy", r.Domain)
- }
- }
- if err := r.Build(ctx, &policy, update); err != nil {
- return nil, nil, err
- }
- if policyID := r.ID(&policy); policyID != "" {
- kept[policyID] = struct{}{}
- }
- policies = append(policies, policy)
- }
- return policies, kept, nil
-}
-
var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")
const RemoteDisabledMessage = "S3 backup storage is missing. Remote backups were disabled; edit the policy to resume."
-// RemoteSnapshotChecker memoizes targeted checks within one response budget.
-// A nil result means the remote copy could not be verified.
-func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationService, root string) func(string, string) *bool {
+// RemoteSnapshotChecker memoizes targeted checks within one response budget; each lookup names the
+// repository root that holds the snapshot. A nil result means the remote copy could not be verified.
+func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationService) func(destinationID, root, snapshotID string) *bool {
checked := make(map[string]*backup.RepositoryObservation)
deadline := time.Now().Add(10 * time.Second)
- return func(destinationID, snapshotID string) *bool {
+ return func(destinationID, root, snapshotID string) *bool {
if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {
return nil
}
- key := destinationID + ":" + snapshotID
+ key := destinationID + ":" + root + ":" + snapshotID
observation, seen := checked[key]
if !seen {
checked[key] = nil
@@ -894,6 +925,34 @@ func CheckRemoteRepository(ctx context.Context, destinations *s3.S3DestinationSe
return s3util.CheckRepository(checkCtx, configuration, root, "")
}
+// DisableMissingRemote stops a scheduled policy from using S3 storage that went missing: a policy that also backs up
+// locally loses S3, an S3-only one is disabled. save clears that field on the stored policy unless it changed since it
+// was loaded, and reports whether it did; the loaded flags then follow. It reports whether S3 was turned off.
+func DisableMissingRemote(remoteErr error, localEnabled bool, s3Enabled, enabled *bool, save func(field string) (bool, error)) (bool, error) {
+ if !errors.Is(remoteErr, ErrRemoteRepositoryMissing) {
+ return false, nil
+ }
+ field, flag := "enabled", enabled
+ if localEnabled {
+ field, flag = "s3_enabled", s3Enabled
+ }
+ saved, err := save(field)
+ if err != nil || !saved {
+ return false, err
+ }
+ *flag = false
+ return true, nil
+}
+
+// DisableStoredRemote is DisableMissingRemote's save for a policy table row, which it changes only while the row still
+// matches the loaded policy.
+func DisableStoredRemote(ctx context.Context, db *database.DB, model any, id, destinationID string, localEnabled bool, field string) (bool, error) {
+ result := db.WithContext(ctx).Model(model).
+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", id, destinationID, true, true, localEnabled).
+ Update(field, false)
+ return result.Error == nil && result.RowsAffected > 0, result.Error
+}
+
// CheckScheduledRemote permits first-use initialization, but detects lost repositories.
func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3.S3DestinationService, table, destinationID, root string) error {
result, err := CheckRemoteRepository(ctx, destinations, destinationID, root)
@@ -914,371 +973,6 @@ func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3
return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)
}
-const backupRunTypeInternal = "backup-run"
-
-type backupRunActorInternal struct {
- id string
- engine *Engine
- service *actor.Service
-}
-
-// RegisterRunKind binds a domain handler before the actor host starts.
-func (e *Engine) RegisterRunKind(kind string, execute func(context.Context, string, []byte, bool) error, failures ...func(context.Context, string, []byte, error) error) {
- e.mu.Lock()
- defer e.mu.Unlock()
- e.handlers[kind] = execute
- if len(failures) > 0 {
- e.failures[kind] = failures[0]
- }
-}
-
-func (e *Engine) SetAuthorize(authorize func(context.Context, backup.DurableRunCommand) error) {
- e.authorize = authorize
-}
-
-func (e *Engine) SetExecutionReady(ready func() bool) { e.executionReady = ready }
-
-func (e *Engine) Register(runtime *francis.Runtime) error {
- e.service = runtime.Service()
- return runtime.RegisterActor(backupRunTypeInternal, func(id string, service *actor.Service) actor.Actor {
- return &backupRunActorInternal{id: id, engine: e, service: service}
- }, local.WithCapacityGroup("jobs", 4), local.WithCompletedJobRetention(7*24*time.Hour))
-}
-
-// SubmitDurableRun persists the accepted command before dispatching its work.
-// A nil error means the run was accepted or its outcome is still unresolved;
-// an error means the intent was confirmed absent and admission is the caller's.
-func (e *Engine) SubmitDurableRun(ctx context.Context, command backup.DurableRunCommand, lease *runs.Lease) error {
- if e == nil || e.service == nil {
- return errors.New("backup actor host is unavailable")
- }
- if e.authorize != nil {
- if err := e.authorize(ctx, command); err != nil {
- return err
- }
- }
- e.mu.Lock()
- if e.stopping {
- e.mu.Unlock()
- return errors.New("backup engine is stopping")
- }
- e.leases[command.RunID] = lease
- e.mu.Unlock()
- ctx = context.WithoutCancel(ctx)
- _, err := e.service.Invoke(ctx, backupRunTypeInternal, command.RunID, "submit", command)
- if err == nil {
- return nil
- }
- var state backup.DurableRunState
- readErr := e.service.GetState(ctx, backupRunTypeInternal, command.RunID, &state)
- switch {
- case readErr == nil && sameDurableCommandInternal(state.Command, command):
- slog.WarnContext(ctx, "backup run accepted after submit error", "runId", command.RunID, "error", err)
- return nil
- case readErr == nil || errors.Is(readErr, actor.ErrStateNotFound):
- e.mu.Lock()
- delete(e.leases, command.RunID)
- e.mu.Unlock()
- return err
- default:
- // Keep admission parked until reconciliation can read the intent back.
- slog.WarnContext(ctx, "backup run outcome unresolved", "runId", command.RunID, "error", err, "readError", readErr)
- e.mu.Lock()
- e.unresolved[command.RunID] = command
- e.mu.Unlock()
- return nil
- }
-}
-
-// AcquireDurableRun keeps the original admission, or reacquires it after a restart.
-func (e *Engine) AcquireDurableRun(ctx context.Context, runID, scope, resourceID string) (*runs.Lease, bool, error) {
- e.mu.Lock()
- lease := e.leases[runID]
- delete(e.leases, runID)
- e.mu.Unlock()
- if lease != nil {
- return lease, true, nil
- }
- return e.TryAcquireRun(ctx, scope, resourceID)
-}
-
-func (a *backupRunActorInternal) Invoke(ctx context.Context, _ string, data actor.Envelope) (any, error) {
- var command backup.DurableRunCommand
- if err := data.Decode(&command); err != nil {
- return nil, err
- }
- var state backup.DurableRunState
- err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state)
- switch {
- case errors.Is(err, actor.ErrStateNotFound):
- state = backup.DurableRunState{Command: command, Status: scheduler.Queued}
- if setStateErr := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); setStateErr != nil {
- return nil, setStateErr
- }
- case err != nil:
- return nil, err
- case !sameDurableCommandInternal(state.Command, command):
- return nil, errors.New("backup run already exists with a different command")
- }
- if state.Status == scheduler.Succeeded || state.Status == scheduler.Failed || state.Status == scheduler.NeedsAttention {
- return nil, nil
- }
- // The intent is persisted, so ReconcileDispatches repairs a failed dispatch.
- if _, _, dispatchErr := a.service.Dispatch(ctx, backupRunTypeInternal, a.id, "execute", nil, actor.WithIdempotencyKey(a.id)); dispatchErr != nil {
- slog.WarnContext(ctx, "dispatch accepted backup run", "runId", a.id, "error", dispatchErr)
- }
- return nil, nil
-}
-
-func sameDurableCommandInternal(a, b backup.DurableRunCommand) bool {
- return a.UserID == b.UserID &&
- a.EnvironmentID == b.EnvironmentID &&
- a.Permission == b.Permission &&
- a.RequestedWithKey == b.RequestedWithKey &&
- a.Kind == b.Kind &&
- a.RunID == b.RunID &&
- a.ActivityID == b.ActivityID &&
- bytes.Equal(a.Payload, b.Payload)
-}
-
-func (a *backupRunActorInternal) Job(ctx context.Context, _ string, _ actor.Envelope) error {
- if a.engine.executionReady != nil && !a.engine.executionReady() {
- return actor.ErrJobRejected
- }
- a.engine.mu.Lock()
- if a.engine.stopping {
- a.engine.mu.Unlock()
- return actor.ErrJobRejected
- }
- a.engine.workers.Add(1)
- a.engine.mu.Unlock()
- defer a.engine.workers.Done()
- jobCtx, cancel := context.WithCancel(ctx)
- stop := context.AfterFunc(a.engine.lifecycleCtx, cancel) //nolint:contextcheck // Cancels the inherited job context when the engine stops.
- defer stop()
- defer cancel()
- ctx = jobCtx
-
- var state backup.DurableRunState
- if err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state); err != nil {
- return err
- }
- if state.Status == scheduler.Succeeded || state.Status == scheduler.Failed || state.Status == scheduler.NeedsAttention {
- return nil
- }
- a.engine.mu.Lock()
- pendingLease := a.engine.leases[a.id]
- a.engine.mu.Unlock()
- defer pendingLease.Release(ctx)
- interrupted := state.Started && len(state.Targets) > 0
- state.Started = true
- state.Status = scheduler.Running
- if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {
- return err
- }
- var progressMu sync.Mutex
- ctx = jobcontext.WithExecution(ctx, scheduler.Run{ID: a.id, EnvironmentID: "0", Outcome: scheduler.Outcome{Targets: state.Targets}}, func(target scheduler.TargetOutcome) error {
- progressMu.Lock()
- defer progressMu.Unlock()
- updated := false
- for i := range state.Targets {
- if state.Targets[i].ID == target.ID {
- if len(target.RecoveryData) == 0 {
- target.RecoveryData = state.Targets[i].RecoveryData
- }
- state.Targets[i] = target
- updated = true
- break
- }
- }
- if !updated {
- state.Targets = append(state.Targets, target)
- }
- return a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil)
- })
- runErr := a.executeInternal(ctx, state.Command, interrupted)
- if ctx.Err() != nil {
- return ctx.Err()
- }
- state.Status = scheduler.Succeeded
- if runErr != nil {
- state.Status = scheduler.NeedsAttention
- state.Error = runErr.Error()
- }
- if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {
- return err
- }
- if runErr != nil {
- return fmt.Errorf("%s: %w", runErr.Error(), actor.ErrJobPermanentFailure)
- }
- return nil
-}
-
-func (a *backupRunActorInternal) executeInternal(ctx context.Context, command backup.DurableRunCommand, interrupted bool) (err error) {
- defer utils.RecoverToError(&err, "durable backup")
- a.engine.mu.Lock()
- handler := a.engine.handlers[command.Kind]
- failure := a.engine.failures[command.Kind]
- a.engine.mu.Unlock()
- if handler == nil {
- return errors.New("backup run kind is unavailable")
- }
- if a.engine.authorize != nil {
- if authorizeErr := a.engine.authorize(ctx, command); authorizeErr != nil {
- a.engine.mu.Lock()
- lease := a.engine.leases[a.id]
- delete(a.engine.leases, a.id)
- a.engine.mu.Unlock()
- defer lease.Release(ctx)
- if failure != nil {
- authorizeErr = errors.Join(authorizeErr, failure(ctx, a.id, command.Payload, authorizeErr))
- }
- return authorizeErr
- }
- }
- return handler(ctx, a.id, command.Payload, interrupted)
-}
-
-func (e *Engine) activeRunsInternal(ctx context.Context) ([]backup.DurableRunState, error) {
- result := []backup.DurableRunState{}
- for cursor := ""; ; {
- page, err := e.service.ListStates(ctx, backupRunTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
- if err != nil {
- return nil, err
- }
- for _, item := range page.States {
- if item.Data == nil {
- continue
- }
- var state backup.DurableRunState
- if decodeErr := item.Data.Decode(&state); decodeErr != nil {
- return nil, decodeErr
- }
- if state.Status == scheduler.Queued || state.Status == scheduler.Running || state.Status == scheduler.NeedsAttention {
- result = append(result, state)
- }
- }
- cursor = page.AfterID()
- if cursor == "" {
- break
- }
- }
- return result, nil
-}
-
-func (e *Engine) ActiveRunIDs(ctx context.Context) ([]string, error) {
- states, err := e.activeRunsInternal(ctx)
- if err != nil {
- return nil, err
- }
- ids := make([]string, 0, len(states))
- for _, state := range states {
- ids = append(ids, state.Command.RunID)
- for _, target := range state.Targets {
- var checkpoint struct {
- BackupID string `json:"backupId"`
- }
- if len(target.RecoveryData) > 0 && json.Unmarshal(target.RecoveryData, &checkpoint) == nil && checkpoint.BackupID != "" {
- ids = append(ids, checkpoint.BackupID)
- }
- }
- }
- return ids, nil
-}
-
-func (e *Engine) ActiveActivityIDs(ctx context.Context) ([]string, error) {
- states, err := e.activeRunsInternal(ctx)
- if err != nil {
- return nil, err
- }
- ids := make([]string, 0, len(states))
- for _, state := range states {
- if state.Command.ActivityID != "" {
- ids = append(ids, state.Command.ActivityID)
- }
- }
- return ids, nil
-}
-
-// ReconcileDispatches resolves unresolved submissions and repairs accepted
-// commands whose dispatch was interrupted.
-func (e *Engine) ReconcileDispatches(ctx context.Context) error {
- failures := e.resolveUnresolvedRunsInternal(ctx)
- states, err := e.activeRunsInternal(ctx)
- if err != nil {
- return errors.Join(append(failures, err)...)
- }
- for _, state := range states {
- if state.Status == scheduler.NeedsAttention {
- continue
- }
- if repairErr := e.repairDispatchInternal(ctx, state.Command.RunID); repairErr != nil {
- failures = append(failures, fmt.Errorf("repair backup run %s: %w", state.Command.RunID, repairErr))
- }
- }
- return errors.Join(failures...)
-}
-
-func (e *Engine) repairDispatchInternal(ctx context.Context, runID string) error {
- jobs, err := e.service.ListJobs(ctx, backupRunTypeInternal, runID)
- if err != nil {
- return err
- }
- for _, job := range jobs {
- if !job.Status.IsTerminal() {
- return nil
- }
- if job.Status == actor.JobStatusDeadLettered {
- _, retryJobErr := e.service.RetryJob(ctx, job.JobID)
- return retryJobErr
- }
- if deleteJobErr := e.service.DeleteJob(ctx, backupRunTypeInternal, runID, job.JobID); deleteJobErr != nil && !errors.Is(deleteJobErr, actor.ErrJobNotFound) {
- return deleteJobErr
- }
- }
- _, _, err = e.service.Dispatch(ctx, backupRunTypeInternal, runID, "execute", nil, actor.WithIdempotencyKey(runID))
- return err
-}
-
-// resolveUnresolvedRunsInternal reads back submissions whose outcome was
-// unknown. Found intents continue as accepted; confirmed absence releases
-// admission and fails the run through its kind's failure handler.
-func (e *Engine) resolveUnresolvedRunsInternal(ctx context.Context) []error {
- e.mu.Lock()
- pending := maps.Clone(e.unresolved)
- e.mu.Unlock()
- var failures []error
- for runID, command := range pending {
- var state backup.DurableRunState
- readErr := e.service.GetState(ctx, backupRunTypeInternal, runID, &state)
- if readErr != nil && !errors.Is(readErr, actor.ErrStateNotFound) {
- failures = append(failures, fmt.Errorf("read back backup run %s: %w", runID, readErr))
- continue
- }
- e.mu.Lock()
- lease := e.leases[runID]
- failure := e.failures[command.Kind]
- if readErr != nil {
- delete(e.leases, runID)
- }
- e.mu.Unlock()
- if readErr != nil {
- lease.Release(ctx)
- // Stay unresolved until the failure is recorded, so a later tick retries it.
- if failure != nil {
- if failErr := failure(ctx, runID, command.Payload, errors.New("backup run was not accepted")); failErr != nil {
- failures = append(failures, fmt.Errorf("fail backup run %s: %w", runID, failErr))
- continue
- }
- }
- }
- e.mu.Lock()
- delete(e.unresolved, runID)
- e.mu.Unlock()
- }
- return failures
-}
-
// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall
// outside the newest keep entries for one policy, oldest last. Callers delete
// each run through their own delete path so snapshots are forgotten too.
diff --git a/backend/internal/backup/service_test.go b/backend/internal/backup/service_test.go
index 4b363f31f..c30d2edab 100644
--- a/backend/internal/backup/service_test.go
+++ b/backend/internal/backup/service_test.go
@@ -1,14 +1,8 @@
package backup
import (
- "context"
- "path/filepath"
- "sync/atomic"
"testing"
- "time"
- "github.com/getarcaneapp/arcane/types/v2/backup"
- "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/libtnb/sqlite"
"github.com/moby/moby/api/types/mount"
"github.com/stretchr/testify/require"
@@ -16,29 +10,26 @@ import (
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func TestMarkSnapshotDirectoriesInternal(t *testing.T) {
+func TestMarkSnapshotDirectories(t *testing.T) {
files := []string{"/volume", "/volume/folder", "/volume/file.txt", "/volume/link"}
longOutput := "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"\r\ndrwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume/" +
"folder\"\r\n-rw-r--r-- root root 5 1 Jan 2026 00:00 \"/volume/file.txt\"\r\nlrwxrwxrwx root root 4 1 Jan 20" +
"26 00:00 \"/volume/link\" -> \"file.txt\""
- marked, err := markSnapshotDirectoriesInternal(files, longOutput)
+ marked, err := markSnapshotDirectories(files, longOutput)
require.NoError(t, err)
require.Equal(t, []string{"/volume/", "/volume/folder/", "/volume/file.txt", "/volume/link"}, marked)
require.Equal(t, []string{"/volume", "/volume/folder", "/volume/file.txt", "/volume/link"}, files)
}
-func TestMarkSnapshotDirectoriesRejectsMismatchedListingsInternal(t *testing.T) {
- _, err := markSnapshotDirectoriesInternal([]string{"/volume", "/volume/file.txt"}, "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"")
+func TestMarkSnapshotDirectoriesRejectsMismatchedListings(t *testing.T) {
+ _, err := markSnapshotDirectories([]string{"/volume", "/volume/file.txt"}, "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"")
require.ErrorContains(t, err, "different lengths")
}
-func TestQualifySnapshotListingInternal(t *testing.T) {
+func TestQualifySnapshotListing(t *testing.T) {
tests := []struct {
name string
files []string
@@ -60,19 +51,19 @@ func TestQualifySnapshotListingInternal(t *testing.T) {
{
name: "nested listing",
files: []string{"nested/", "file.txt", "./link"},
- snapshotPath: "folder/",
+ snapshotPath: "folder",
expected: []string{"folder/nested/", "folder/file.txt", "folder/link"},
},
{
name: "legacy system project listing",
files: []string{"demo/compose.yaml"},
- snapshotPath: "/app/data/projects/",
+ snapshotPath: "app/data/projects",
expected: []string{"app/data/projects/demo/compose.yaml"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
- qualified := qualifySnapshotListingInternal(test.files, test.snapshotPath)
+ qualified := qualifySnapshotListing(test.files, test.snapshotPath)
require.Equal(t, test.expected, qualified)
if len(test.files) > 0 {
require.NotSame(t, &test.files[0], &qualified[0])
@@ -119,12 +110,12 @@ func TestRecoveryKeyValidation(t *testing.T) {
require.NoError(t, ValidateRecoveryKey(key))
}
-func TestSnapshotCommandInternal(t *testing.T) {
- single, err := snapshotCommandInternal("volume", RootSnapshotInput(mount.Mount{Type: mount.TypeVolume, Source: "data", Target: "/volume"}))
+func TestSnapshotCommand(t *testing.T) {
+ single, err := snapshotCommand("volume", RootSnapshotInput(mount.Mount{Type: mount.TypeVolume, Source: "data", Target: "/volume"}))
require.NoError(t, err)
require.Equal(t, []string{"backup", "--init", "--json", "--host", "arcane", "--label", "volume", "--as-path", "/", "--", "/volume"}, single)
- multi, err := snapshotCommandInternal("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, Globs: []string{"!/data/arcane.db-wal"}})
+ multi, err := snapshotCommand("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, Globs: []string{"!/data/arcane.db-wal"}})
require.NoError(t, err)
require.Equal(
t,
@@ -145,111 +136,8 @@ func TestSnapshotCommandInternal(t *testing.T) {
multi,
)
- _, err = snapshotCommandInternal("x", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, AsPath: "/"})
+ _, err = snapshotCommand("x", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, AsPath: "/"})
require.ErrorContains(t, err, "single source")
- _, err = snapshotCommandInternal("x", CreateSnapshotInput{})
+ _, err = snapshotCommand("x", CreateSnapshotInput{})
require.ErrorContains(t, err, "at least one snapshot source")
}
-
-func TestDurableBackupRestartKeepsCompletionInternal(t *testing.T) {
- url := "file:" + filepath.Join(t.TempDir(), "actors.db")
- var calls atomic.Int32
- runtime := francistest.New(t, url)
- engine := NewEngine(t.Context(), nil, nil)
- engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {
- calls.Add(1)
- require.False(t, interrupted)
- return jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "snapshot", Status: scheduler.Succeeded})
- })
- require.NoError(t, engine.Register(runtime))
- francistest.Start(t, runtime)
- command := backup.DurableRunCommand{Kind: "test", RunID: "run", ActivityID: "activity"}
- require.NoError(t, engine.SubmitDurableRun(t.Context(), command, nil))
- require.Eventually(t, func() bool {
- states, err := engine.activeRunsInternal(t.Context())
- return err == nil && len(states) == 0 && calls.Load() == 1
- }, 10*time.Second, 20*time.Millisecond)
- stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
- defer cancel()
- require.NoError(t, runtime.Stop(stopCtx))
- restarted := francistest.New(t, url)
- next := NewEngine(t.Context(), nil, nil)
- next.RegisterRunKind("test", func(context.Context, string, []byte, bool) error { calls.Add(1); return nil })
- require.NoError(t, next.Register(restarted))
- francistest.Start(t, restarted)
- require.NoError(t, next.SubmitDurableRun(t.Context(), command, nil))
- require.NoError(t, next.ReconcileDispatches(t.Context()))
- require.EqualValues(t, 1, calls.Load())
-}
-
-func TestDurableBackupShutdownKeepsInterruptedEvidenceInternal(t *testing.T) {
- url := "file:" + filepath.Join(t.TempDir(), "actors.db")
- runtime := francistest.New(t, url)
- admission := runs.NewAdmission(runtime.Service(), t.Name())
- require.NoError(t, admission.Register(runtime))
- engine := NewEngine(t.Context(), admission, nil)
- entered := make(chan struct{})
- drain := make(chan struct{})
- t.Cleanup(func() {
- select {
- case <-drain:
- default:
- close(drain)
- }
- })
- engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, _ bool) error {
- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "snapshot", Status: scheduler.Running, RecoveryData: []byte(`{"backupId":"frozen"}`)}); err != nil {
- return err
- }
- close(entered)
- <-ctx.Done()
- <-drain
- return ctx.Err()
- })
- require.NoError(t, engine.Register(runtime))
- francistest.Start(t, runtime)
- lease, acquired, err := engine.TryAcquireRun(t.Context(), "backup", "data")
- require.NoError(t, err)
- require.True(t, acquired)
- require.NoError(t, engine.SubmitDurableRun(t.Context(), backup.DurableRunCommand{Kind: "test", RunID: "interrupted"}, lease))
- select {
- case <-entered:
- case <-time.After(10 * time.Second):
- t.Fatal("backup did not start")
- }
- stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
- defer cancel()
- stopped := make(chan error, 1)
- go func() { stopped <- engine.Stop(stopCtx) }()
- require.Eventually(t, func() bool { return engine.lifecycleCtx.Err() != nil }, time.Second, time.Millisecond)
- _, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")
- require.NoError(t, err)
- require.False(t, acquired)
- close(drain)
- require.NoError(t, <-stopped)
- lease, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")
- require.NoError(t, err)
- require.True(t, acquired)
- lease.Release(t.Context())
- require.NoError(t, runtime.Stop(stopCtx))
- restarted := francistest.New(t, url)
- next := NewEngine(t.Context(), nil, nil)
- resumed := make(chan struct{})
- next.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {
- require.True(t, interrupted)
- previous, ok := jobcontext.Run(ctx)
- require.True(t, ok)
- require.Len(t, previous.Outcome.Targets, 1)
- require.JSONEq(t, `{"backupId":"frozen"}`, string(previous.Outcome.Targets[0].RecoveryData))
- close(resumed)
- return nil
- })
- require.NoError(t, next.Register(restarted))
- francistest.Start(t, restarted)
- require.NoError(t, next.ReconcileDispatches(t.Context()))
- select {
- case <-resumed:
- case <-time.After(15 * time.Second):
- t.Fatal("durable backup did not resume")
- }
-}
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index dc6e756ee..25e245226 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -33,6 +33,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
@@ -55,7 +56,7 @@ func (w *blockingBusWatcherInternal) Start(ctx context.Context) error {
func (w *blockingBusWatcherInternal) RunNow(context.Context) error { return nil }
-func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {
+func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {
fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName
t.Run("nil request", func(t *testing.T) {
@@ -111,7 +112,7 @@ func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {
})
}
-func TestIsTunnelGRPCRequestInternal(t *testing.T) {
+func TestIsTunnelGRPCRequest(t *testing.T) {
fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName
t.Run("detects by grpc content-type", func(t *testing.T) {
@@ -160,7 +161,7 @@ func TestIsTunnelGRPCRequestInternal(t *testing.T) {
})
}
-func TestConfigureHTTPProtocolsInternal(t *testing.T) {
+func TestConfigureHTTPProtocols(t *testing.T) {
handler := http.NewServeMux()
t.Run("tls enables http1 and http2", func(t *testing.T) {
@@ -186,7 +187,7 @@ func TestConfigureHTTPProtocolsInternal(t *testing.T) {
})
}
-func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
+func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {
cfg := &config.Config{
AgentMode: true,
AppUrl: "http://localhost:3552",
@@ -199,7 +200,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
Project: project.New(&project.ProjectService{}, nil),
Container: container.New(&container.ContainerService{}, nil, nil, nil),
Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
- System: system.New(&system.SystemService{}, nil, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil, nil),
},
AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
TunnelRegistry: edge.NewTunnelRegistry(),
@@ -254,7 +255,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
}
}
-func TestH2CStreamSurvivesPastReadHeaderTimeoutInternal(t *testing.T) {
+func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {
// Regression guard for the go1.26.6 CVE-2026-56853 backport, which arms
// ReadHeaderTimeout on the raw conn before the h2c preface sniff and never
// clears it on the HTTP/2 handoff. Without ClearReadDeadline in the h2c
@@ -310,7 +311,7 @@ func TestH2CStreamSurvivesPastReadHeaderTimeoutInternal(t *testing.T) {
require.Equal(t, "done", string(body))
}
-func TestHTTPServerStopCancelsStreamingRequestContextsInternal(t *testing.T) {
+func TestHTTPServerStopCancelsStreamingRequestContexts(t *testing.T) {
appCtx := t.Context()
handlerEntered := make(chan struct{})
@@ -350,7 +351,7 @@ func TestHTTPServerStopCancelsStreamingRequestContextsInternal(t *testing.T) {
require.Less(t, time.Since(start), time.Second)
}
-func TestNewHTTPServerRejectsInvalidTLSCertificateInternal(t *testing.T) {
+func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {
cfg := &config.Config{
AgentMode: true,
TLSEnabled: true,
@@ -384,7 +385,7 @@ func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {
require.ErrorIs(t, appCtx.Err(), context.Canceled)
}
-func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
+func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {
appCtx := t.Context()
lifecycle := fxtest.NewLifecycle(t)
@@ -400,6 +401,8 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
require.NoError(t, coordinator.Register(runtime))
admission := runs.NewAdmission(runtime.Service(), t.Name())
require.NoError(t, admission.Register(runtime))
+ workflows, err := flow.New(appCtx, runtime, coordinator, nil)
+ require.NoError(t, err)
lifecycle.Append(fx.Hook{
OnStart: func(ctx context.Context) error { return runtime.Start(ctx, appCtx, nil) },
OnStop: runtime.Stop,
@@ -409,7 +412,7 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
gitopsSync := gitops.NewGitOpsSyncService(&database.DB{DB: db}, nil, nil, nil, nil, nil)
require.NoError(t, db.Create(&environment.Environment{ID: "0", Name: "Local", Enabled: true}).Error)
require.NoError(t, db.Create(&project.GitOpsSync{ID: "overdue", EnvironmentID: "0", AutoSync: true, SyncInterval: 1}).Error)
- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, nil, nil, nil, gitopsSync)
+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, workflows, nil, gitopsSync, runtime)
require.NoError(t, err)
require.NoError(t, registerDynamicJobs(dynamicJobsParams{
AppCtx: appCtx, Config: &config.Config{}, Scheduler: jobScheduler,
@@ -507,7 +510,7 @@ func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {
require.FileExists(t, assetsDir+"/ca.key")
}
-func TestIsWeakProductionEncryptionKeyInternal(t *testing.T) {
+func TestIsWeakProductionEncryptionKey(t *testing.T) {
assert.True(t, isWeakProductionEncryptionKeyInternal("short", "production", false))
assert.False(t, isWeakProductionEncryptionKeyInternal("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))
assert.False(t, isWeakProductionEncryptionKeyInternal("hex:abc", "production", false))
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index b7148f540..19143e001 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -1,6 +1,7 @@
package bootstrap
import (
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -9,26 +10,26 @@ import (
"net/http"
"slices"
"sync"
- "time"
"github.com/getarcaneapp/arcane/types/v2/features"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
"go.uber.org/fx"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
- "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
- "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
func newJobScheduler(
@@ -41,10 +42,10 @@ func newJobScheduler(
analytics *scheduler.AnalyticsJob,
systemService *system.SystemService,
jobService *job.JobService,
- backupEngine *backup.Engine,
- updaterService *updater.UpdaterService,
+ workflows *flow.Engine,
volumes *volume.VolumeService,
gitopsSync *gitops.GitOpsSyncService,
+ actors *francis.Runtime,
) (
schedulertypes.JobScheduler,
error,
@@ -56,7 +57,6 @@ func newJobScheduler(
return nil, err
}
jobService.SetScheduler(schedulerCtx, jobScheduler)
- var repairDone chan struct{}
var resumeDone chan struct{}
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
@@ -70,9 +70,10 @@ func newJobScheduler(
if startErr := jobService.Coordinator().Start(ctx, schedulerCtx); startErr != nil {
return fmt.Errorf("initialize scheduler: %w", startErr)
}
- if reconcileCoordinatorStartupErr := reconcileCoordinatorStartupInternal(ctx, jobService, backupEngine, updaterService, volumes, systemService, gitopsSync); reconcileCoordinatorStartupErr != nil {
+ if reconcileCoordinatorStartupErr := reconcileCoordinatorStartup(ctx, jobService, workflows, volumes, systemService, gitopsSync); reconcileCoordinatorStartupErr != nil {
return fmt.Errorf("reconcile scheduler startup: %w", reconcileCoordinatorStartupErr)
}
+ settleLegacyActorWork(ctx, actors.Service())
if gitopsSync != nil {
gitopsSync.RegisterAutoSyncJobsOnStartup(ctx)
gitopsSync.SubscribeProjectFileChanges(schedulerCtx)
@@ -87,21 +88,22 @@ func newJobScheduler(
return fmt.Errorf("start scheduler: %w", startSchedulerErr)
}
jobService.Coordinator().Activate()
- if backupEngine != nil {
- repairDone = make(chan struct{})
- go func() {
- defer close(repairDone)
- repairBackupDispatchesInternal(schedulerCtx, backupEngine)
- }()
- }
if analytics != nil {
if _, submitErr := jobService.Coordinator().Submit(ctx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); submitErr != nil {
return fmt.Errorf("submit startup job %q: %w", analytics.Name(), submitErr)
}
}
+ // Standalone tasks wait until update-all recovery has read its job, so a resumed step's new checkpoint
+ // is never mistaken for one the previous process left.
if !cfg.AgentMode && systemService != nil {
resumeDone = make(chan struct{})
- go func() { defer close(resumeDone); systemService.ResumeUpdateAllOnStartup(schedulerCtx) }()
+ go func() {
+ defer close(resumeDone)
+ systemService.ResumeUpdateAllOnStartup(schedulerCtx)
+ workflows.Ready()
+ }()
+ } else {
+ workflows.Ready()
}
if startupCancellationErr := ctx.Err(); startupCancellationErr != nil {
return fmt.Errorf("complete scheduler startup: %w", startupCancellationErr)
@@ -110,138 +112,125 @@ func newJobScheduler(
return nil
},
OnStop: func(ctx context.Context) error {
- return stopJobSchedulerInternal(ctx, cancelScheduler, []chan struct{}{repairDone, resumeDone}, jobService.Coordinator(), jobScheduler)
+ cancelScheduler()
+ if resumeDone != nil {
+ select {
+ case <-resumeDone:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ if stopErr := errors.Join(jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {
+ slog.ErrorContext(ctx, "Job scheduler exited with error", "error", stopErr)
+ return stopErr
+ }
+ slog.InfoContext(ctx, "Scheduler stopped")
+ return nil
},
})
return jobScheduler, nil
}
-func stopJobSchedulerInternal(ctx context.Context, cancelScheduler context.CancelFunc, workers []chan struct{}, coordinator *runs.Coordinator, jobScheduler schedulertypes.JobScheduler) error {
- cancelScheduler()
- for _, done := range workers {
- if done != nil {
- select {
- case <-done:
- case <-ctx.Done():
- return ctx.Err()
+// legacyActorWork names the state and job actor types of the actors workflows replaced.
+// TODO(v3): remove once no supported upgrade path can carry their saved work.
+var legacyActorWork = []struct{ stateType, jobType, jobActorID string }{
+ {stateType: "backup-run", jobType: "backup-run"},
+ {stateType: "single-container-update-state", jobType: "single-container-update", jobActorID: "single-container"},
+}
+
+// settleLegacyActorWork removes the saved commands and pending jobs of the replaced backup and container-update
+// actors. Nothing delivers them any more; the startup reconcile above has already failed their records and activities.
+func settleLegacyActorWork(ctx context.Context, service *actor.Service) {
+ for _, legacy := range legacyActorWork {
+ settled := 0
+ for cursor := ""; ; {
+ page, err := service.ListStates(ctx, legacy.stateType, &actor.ListStatesOpts{After: cursor, Limit: 100})
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to list work saved by a replaced actor", "actorType", legacy.stateType, "error", err)
+ break
+ }
+ for _, state := range page.States {
+ jobs, listErr := service.ListJobs(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID))
+ for _, job := range jobs {
+ listErr = errors.Join(listErr, service.DeleteJob(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID), job.JobID))
+ }
+ if settleErr := errors.Join(listErr, service.DeleteState(ctx, legacy.stateType, state.ActorID)); settleErr != nil && !errors.Is(settleErr, actor.ErrStateNotFound) {
+ slog.WarnContext(ctx, "Failed to settle work saved by a replaced actor", "actorType", legacy.stateType, "actorId", state.ActorID, "error", settleErr)
+ continue
+ }
+ settled++
+ }
+ if cursor = page.AfterID(); cursor == "" {
+ break
}
}
+ if settled > 0 {
+ slog.InfoContext(ctx, "Settled work saved by a replaced actor; interrupted backups and updates were marked failed", "actorType", legacy.stateType, "count", settled)
+ }
}
- err := errors.Join(coordinator.Stop(ctx), jobScheduler.Stop(ctx))
- if err != nil {
- slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)
- return err
- }
- slog.InfoContext(ctx, "Scheduler stopped")
- return nil
}
-func reconcileCoordinatorStartupInternal(
+// reconcileCoordinatorStartup fails backups and activities the previous process left running,
+// except those that unfinished runs and standalone workflows still own.
+func reconcileCoordinatorStartup(
ctx context.Context,
jobs *job.JobService,
- engine *backup.Engine,
- updates *updater.UpdaterService,
+ workflows *flow.Engine,
volumes *volume.VolumeService,
systemService *system.SystemService,
syncService *gitops.GitOpsSyncService,
) error {
- activities, backups, err := startupProtectionInternal(ctx, jobs, engine, updates)
- if err != nil {
- return err
- }
- if volumes != nil {
- if reconcileInterruptedBackupsErr := volumes.ReconcileInterruptedBackups(ctx, backups...); reconcileInterruptedBackupsErr != nil {
- return reconcileInterruptedBackupsErr
- }
- }
- if systemService != nil {
- if reconcileSystemBackupsErr := systemService.ReconcileInterruptedBackups(ctx, backups...); reconcileSystemBackupsErr != nil {
- return reconcileSystemBackupsErr
- }
- }
- if syncService != nil {
- if reconcileInterruptedBackupsOnStartupErr := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); reconcileInterruptedBackupsOnStartupErr != nil {
- return reconcileInterruptedBackupsOnStartupErr
- }
- }
- return jobs.ReconcileStartupActivities(ctx, activities...)
-}
-
-func startupProtectionInternal(ctx context.Context, jobs *job.JobService, engine *backup.Engine, updates *updater.UpdaterService) ([]string, []string, error) {
records, err := jobs.Coordinator().Records(ctx)
if err != nil {
- return nil, nil, err
+ return err
}
- scheduled, err := scheduledBackupProtectionInternal(records)
+ standalone, err := workflows.Standalone(ctx)
if err != nil {
- return nil, nil, err
- }
-
- activities := []string{}
- backups := make([]string, 0, len(scheduled))
- if engine != nil {
- var loadActivityIDsErr error
- activities, loadActivityIDsErr = engine.ActiveActivityIDs(ctx)
- if loadActivityIDsErr != nil {
- return nil, nil, loadActivityIDsErr
- }
- backups, loadActivityIDsErr = engine.ActiveRunIDs(ctx)
- if loadActivityIDsErr != nil {
- return nil, nil, loadActivityIDsErr
- }
- if reconcileDispatchesErr := engine.ReconcileDispatches(ctx); reconcileDispatchesErr != nil {
- return nil, nil, reconcileDispatchesErr
- }
- }
- if updates != nil {
- ids, activeUpdateActivityIDsErr := updates.ActiveUpdateActivityIDs(ctx)
- if activeUpdateActivityIDsErr != nil {
- return nil, nil, activeUpdateActivityIDsErr
- }
- activities = append(activities, ids...)
+ return err
}
- return activities, append(backups, scheduled...), nil
-}
-
-func scheduledBackupProtectionInternal(records []schedulertypes.QueueRecord) ([]string, error) {
- ids := []string{}
- for _, record := range records {
+ activities, backups := []string{}, []string{}
+ for _, record := range append(records, schedulertypes.QueueRecord{Runs: standalone}) {
for _, run := range record.Runs {
if run.Status.Terminal() {
continue
}
for _, target := range run.Outcome.Targets {
- if len(target.RecoveryData) == 0 {
- continue
- }
var checkpoint struct {
BackupID string `json:"backupId"`
}
- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
- return nil, fmt.Errorf("decode startup recovery evidence: %w", err)
+ if len(target.RecoveryData) == 0 {
+ continue
+ }
+ if decodeErr := json.Unmarshal(target.RecoveryData, &checkpoint); decodeErr != nil {
+ return fmt.Errorf("decode startup recovery evidence: %w", decodeErr)
}
if checkpoint.BackupID != "" {
- ids = append(ids, checkpoint.BackupID)
+ backups = append(backups, checkpoint.BackupID)
}
}
}
}
- return ids, nil
-}
-
-func repairBackupDispatchesInternal(ctx context.Context, engine *backup.Engine) {
- timer := time.NewTicker(15 * time.Second)
- defer timer.Stop()
- for {
- select {
- case <-ctx.Done():
- return
- case <-timer.C:
- if err := engine.ReconcileDispatches(ctx); err != nil && ctx.Err() == nil {
- slog.ErrorContext(ctx, "Backup dispatch reconciliation failed", "error", err)
- }
+ for _, run := range standalone {
+ if run.ActivityID != "" {
+ activities = append(activities, run.ActivityID)
+ }
+ }
+ if volumes != nil {
+ if reconcileInterruptedBackupsErr := volumes.ReconcileInterruptedBackups(ctx, backups...); reconcileInterruptedBackupsErr != nil {
+ return reconcileInterruptedBackupsErr
+ }
+ }
+ if systemService != nil {
+ if reconcileSystemBackupsErr := systemService.ReconcileInterruptedBackups(ctx, backups...); reconcileSystemBackupsErr != nil {
+ return reconcileSystemBackupsErr
+ }
+ }
+ if syncService != nil {
+ if reconcileInterruptedBackupsOnStartupErr := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); reconcileInterruptedBackupsOnStartupErr != nil {
+ return reconcileInterruptedBackupsOnStartupErr
}
}
+ return jobs.ReconcileStartupActivities(ctx, activities...)
}
type registerJobsParams struct {
@@ -262,7 +251,7 @@ type registerJobsParams struct {
Admission *runs.Admission
Apns *apns.ApnsService
- AutoUpdate *scheduler.AutoUpdateJob
+ AutoUpdate *flow.Job `name:"auto-update"`
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
DockerClientRefresh *scheduler.DockerClientRefreshJob
Analytics *scheduler.AnalyticsJob
@@ -272,9 +261,9 @@ type registerJobsParams struct {
ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJob
ScheduledPrune *scheduler.ScheduledPruneJob
FilesystemWatcher *scheduler.FilesystemWatcherJob
- VulnerabilityScan *scheduler.VulnerabilityScanJob
+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`
VulnerabilityRisk *scheduler.VulnerabilityRiskJob
- AutoPatch *scheduler.AutoPatchJob
+ AutoPatch *flow.Job `name:"auto-patch"`
AutoHeal *scheduler.AutoHealJob
ActivitySweep *scheduler.ActivitySweepJob
UploadSessionsCleanup *scheduler.UploadSessionsCleanupJob
@@ -330,7 +319,7 @@ func registerJobs(params registerJobsParams) error {
return err
}
- if err := setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
+ if err := setupSettingsSubscriptions(settingsSubscriptionsParams{
Lifecycle: params.Lifecycle,
LifecycleCtx: params.AppCtx,
Config: params.Config,
@@ -413,36 +402,36 @@ type settingsSubscriptionsParams struct {
Lifecycle fx.Lifecycle
LifecycleCtx context.Context
Config *config.Config
- Scheduler settingsEffectsSchedulerInternal
- Settings settingsChangeSubscriberInternal
- Environment timeoutSettingsEnvironmentInternal
+ Scheduler settingsEffectsScheduler
+ Settings settingsChangeSubscriber
+ Environment timeoutSettingsEnvironment
- AutoUpdate *scheduler.AutoUpdateJob
+ AutoUpdate *flow.Job `name:"auto-update"`
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
FilesystemWatcher *scheduler.FilesystemWatcherJob
ScheduledPrune *scheduler.ScheduledPruneJob
- VulnerabilityScan *scheduler.VulnerabilityScanJob
+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`
VulnerabilityRisk *scheduler.VulnerabilityRiskJob
- AutoPatch *scheduler.AutoPatchJob
+ AutoPatch *flow.Job `name:"auto-patch"`
AutoHeal *scheduler.AutoHealJob
Apns *apns.ApnsService
ApnsOutbox *scheduler.ApnsOutboxJob
}
-type settingsChangeSubscriberInternal interface {
+type settingsChangeSubscriber interface {
SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func()
}
-type settingsEffectsSchedulerInternal interface {
+type settingsEffectsScheduler interface {
RescheduleJob(ctx context.Context, job schedulertypes.Job) error
}
-type timeoutSettingsEnvironmentInternal interface {
+type timeoutSettingsEnvironment interface {
ListRemoteEnvironments(ctx context.Context) ([]environment.Environment, error)
ProxyRequest(ctx context.Context, envID, method, path string, body []byte) ([]byte, int, error)
}
-func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) error {
+func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
unsubscribes := make([]func(), 0, 8)
subscribe := func(keys []string, callback func([]libarcane.SettingUpdate)) {
unsubscribes = append(unsubscribes, params.Settings.SubscribeSettingsChanges(keys, callback))
@@ -454,7 +443,7 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erro
}
})
}
- timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscriptionInternal(params, subscribe)
+ timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscription(params, subscribe)
subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {
if params.ImageUpdateWatcher != nil {
@@ -513,7 +502,7 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erro
return nil
}
-type timeoutSyncWorkerInternal struct {
+type timeoutSyncWorker struct {
mu sync.Mutex
updates [][]libarcane.SettingUpdate
wake chan struct{}
@@ -523,7 +512,7 @@ type timeoutSyncWorkerInternal struct {
closed bool
}
-func (w *timeoutSyncWorkerInternal) Stop(ctx context.Context) error {
+func (w *timeoutSyncWorker) Stop(ctx context.Context) error {
if w == nil {
return nil
}
@@ -539,12 +528,12 @@ func (w *timeoutSyncWorkerInternal) Stop(ctx context.Context) error {
}
}
-func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorkerInternal, context.CancelFunc) {
+func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorker, context.CancelFunc) {
if params.Config.AgentMode {
return nil, nil
}
ctx, cancel := context.WithCancel(params.LifecycleCtx)
- worker := &timeoutSyncWorkerInternal{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}
+ worker := &timeoutSyncWorker{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}
go func() {
defer close(worker.done)
for {
@@ -562,7 +551,7 @@ func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams
updates := worker.updates[0]
worker.updates = worker.updates[1:]
worker.mu.Unlock()
- syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)
+ syncTimeoutSettingsToAgents(ctx, params.Environment, updates)
}
}
}()
@@ -581,8 +570,8 @@ func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams
return worker, cancel
}
-// syncTimeoutSettingsToAgentsInternal syncs timeout settings to all connected remote environments
-func syncTimeoutSettingsToAgentsInternal(ctx context.Context, localEnvironment timeoutSettingsEnvironmentInternal, timeoutSettings []libarcane.SettingUpdate) {
+// syncTimeoutSettingsToAgents syncs timeout settings to all connected remote environments
+func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, timeoutSettings []libarcane.SettingUpdate) {
envs, err := localEnvironment.ListRemoteEnvironments(ctx)
if err != nil {
slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)
diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.go
index b1a7db187..eb2d2df26 100644
--- a/backend/internal/bootstrap/jobs_bootstrap_test.go
+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go
@@ -13,8 +13,11 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
)
type settingsSubscriptionStubInternal struct {
@@ -74,19 +77,20 @@ func (s *timeoutSyncEnvironmentStubInternal) ProxyRequest(ctx context.Context, _
return nil, 0, ctx.Err()
}
-func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
+func TestSettingsTimeoutSyncDoesNotBlockOtherEffects(t *testing.T) {
lifecycle := fxtest.NewLifecycle(t)
settings := &settingsSubscriptionStubInternal{}
localScheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
localEnvironment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}
- require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
+ require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{
Lifecycle: lifecycle,
LifecycleCtx: t.Context(),
Config: &config.Config{},
Scheduler: localScheduler,
Settings: settings,
Environment: localEnvironment,
+ AutoUpdate: &flow.Job{JobName: "auto-update"},
}))
timeoutCallbackDone := make(chan struct{})
@@ -118,19 +122,19 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
require.NoError(t, lifecycle.Stop(stopCtx))
}
-func TestFeatureChangeReschedulesScanAndPatchJobsInternal(t *testing.T) {
+func TestFeatureChangeReschedulesScanAndPatchJobs(t *testing.T) {
lifecycle := fxtest.NewLifecycle(t)
settings := &settingsSubscriptionStubInternal{}
schedulerStub := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
- require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
+ require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{
Lifecycle: lifecycle,
LifecycleCtx: t.Context(),
Config: &config.Config{},
Scheduler: schedulerStub,
Settings: settings,
- VulnerabilityScan: scheduler.NewVulnerabilityScanJob(nil, nil),
+ VulnerabilityScan: scheduler.NewVulnerabilityScanJob(nil, &vulnerability.VulnerabilityService{}, nil),
VulnerabilityRisk: scheduler.NewVulnerabilityRiskJob(nil, nil),
- AutoPatch: scheduler.NewAutoPatchJob(nil, nil),
+ AutoPatch: scheduler.NewAutoPatchJob(nil, &image.ImageService{}, nil),
}))
require.Len(t, settings.featureCallbacks, 3)
for _, callback := range settings.featureCallbacks {
diff --git a/backend/internal/container/handler.go b/backend/internal/container/handler.go
index cf23f9710..0c1188612 100644
--- a/backend/internal/container/handler.go
+++ b/backend/internal/container/handler.go
@@ -19,7 +19,7 @@ import (
"github.com/moby/moby/api/types/network"
"github.com/samber/mo"
"go.getarcane.app/docker/types"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -658,12 +658,7 @@ func (h *ContainerHandler) runContainerActionInternal(ctx context.Context, input
}
activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, cfg.CompleteMessage, nil)
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: cfg.SuccessMessage, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput(cfg.SuccessMessage, activityID), nil
}
func (h *ContainerHandler) CommitContainer(ctx context.Context, input *CommitContainerInput) (*handlerutil.Out[container.CommitResult], error) {
@@ -876,12 +871,7 @@ func (h *ContainerHandler) DeleteContainer(ctx context.Context, input *DeleteCon
}
activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Container deleted", nil)
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Container deleted successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Container deleted successfully", activityID), nil
}
func (h *ContainerHandler) SetAutoUpdate(ctx context.Context, input *SetAutoUpdateInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -898,10 +888,5 @@ func (h *ContainerHandler) SetAutoUpdate(ctx context.Context, input *SetAutoUpda
msg := kit.Ternary(excluded, "Auto-update disabled", "Auto-update enabled")
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: msg},
- },
- }, nil
+ return handlerutil.MessageOutput(msg, ""), nil
}
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 0dfb08e04..7a676e062 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -197,7 +197,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
}).Error)
imageSvc := image.NewImageService(db, nil, nil, nil, nil, nil, nil, nil)
projectSvc := project.NewProjectService(db, settingsSvc, nil, imageSvc, nil, nil, nil, nil, config.Load(), nil, nil)
- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, imageSvc, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))
+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, imageSvc, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))
snapshot, err := svc.GetSnapshot(t.Context(), DashboardActionItemsOptions{}, true)
require.NoError(t, err)
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index 1631ac484..776c1e6d0 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -24,7 +24,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/network"
@@ -56,6 +55,7 @@ import (
var ActorOptions = fx.Options(
fx.Provide(provideActorRuntimeInternal, provideRunCoordinatorInternal),
fx.Provide(provideAdmissionGateInternal),
+ fx.Provide(provideWorkflowEngineInternal),
fx.Provide(provideTunnelRegistryInternal),
)
@@ -81,9 +81,9 @@ var ServiceOptions = fx.Options(
environment.NewEnvironmentService,
apns.NewApnsService,
notification.NewNotificationService,
- vulnerability.NewVulnerabilityService,
- imageupdate.NewImageUpdateService,
- image.NewImageService,
+ provideVulnerabilityServiceInternal,
+ provideImageUpdateServiceInternal,
+ provideImageServiceInternal,
build.NewBuildService,
project.NewLifecycleService,
container.NewContainerService,
@@ -93,16 +93,16 @@ var ServiceOptions = fx.Options(
swarm.NewSwarmService,
template.NewTemplateService,
oidc.NewOidcService,
- system.NewSystemService,
+ provideSystemServiceInternal,
diagnostics.NewDiagnosticsService,
- gitops.NewGitOpsSyncService,
+ provideGitOpsSyncServiceInternal,
variable.NewVariableService,
backup.NewRecoveryKeyStore,
upload.NewUploadService,
auth.NewAuthService,
settings.NewSettingsSearchService,
fx.Annotate(settings.NewSettingsService, fx.OnStop(func(ctx context.Context, service *settings.SettingsService) error { return service.Stop(ctx) })),
- fx.Annotate(volume.NewVolumeService, fx.OnStop(func(ctx context.Context, service *volume.VolumeService) { service.CleanupHelperContainers(ctx) })),
+ provideVolumeServiceInternal,
fx.Annotate(webhook.NewWebhookService,
fx.OnStart(func(ctx context.Context, service *webhook.WebhookService) error { return service.LoadTokenHashes(ctx) }),
fx.OnStop(func(ctx context.Context, service *webhook.WebhookService) {
@@ -163,7 +163,7 @@ var ServiceOptions = fx.Options(
// remain bootstrap concerns because their ordering is application-specific.
var JobOptions = fx.Options(
fx.Provide(
- scheduler.NewAutoUpdateJob,
+ fx.Annotate(scheduler.NewAutoUpdateJob, fx.ResultTags(`name:"auto-update"`)),
scheduler.NewImageUpdateWatcher,
scheduler.NewDockerClientRefreshJob,
scheduler.NewAnalyticsJob,
@@ -172,9 +172,9 @@ var JobOptions = fx.Options(
scheduler.NewExpiredSessionsCleanupJob,
scheduler.NewScheduledPruneJob,
provideFilesystemWatcherJobInternal,
- scheduler.NewVulnerabilityScanJob,
+ fx.Annotate(scheduler.NewVulnerabilityScanJob, fx.ResultTags(`name:"vulnerability-scan"`)),
scheduler.NewVulnerabilityRiskJob,
- scheduler.NewAutoPatchJob,
+ fx.Annotate(scheduler.NewAutoPatchJob, fx.ResultTags(`name:"auto-patch"`)),
scheduler.NewAutoHealJob,
scheduler.NewActivitySweepJob,
scheduler.NewUploadSessionsCleanupJob,
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index d5d91aee5..62067af9a 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -60,6 +60,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
)
@@ -110,7 +111,7 @@ type graphParams struct {
AuthMiddleware *auth.AuthMiddleware
JWKSetManager *oidcjwk.KeySetManager
- AutoUpdate *scheduler.AutoUpdateJob
+ AutoUpdate *flow.Job `name:"auto-update"`
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
DockerClientRefresh *scheduler.DockerClientRefreshJob
Analytics *scheduler.AnalyticsJob
@@ -119,7 +120,7 @@ type graphParams struct {
ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJob
ScheduledPrune *scheduler.ScheduledPruneJob
FilesystemWatcher *scheduler.FilesystemWatcherJob
- VulnerabilityScan *scheduler.VulnerabilityScanJob
+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`
AutoHeal *scheduler.AutoHealJob
}
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 5f8851245..332473f86 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -19,14 +19,17 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/network"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/passkey"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
@@ -34,14 +37,18 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
@@ -66,45 +73,29 @@ func provideSettingsModuleInternal(service *settings.SettingsService, search *se
return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)
}
-func provideBackupEngineInternal(
- ctx context.Context,
- lc fx.Lifecycle,
- admission *runs.Admission,
- imageService *image.ImageService,
- runtime *francis.Runtime,
- coordinator *runs.Coordinator,
- roles *role.RoleService,
- cfg *config.Config,
-) (
- *backup.Engine,
- error,
-) {
- engine := backup.NewEngine(ctx, admission, imageService)
- if err := engine.Register(runtime); err != nil {
- return nil, err
- }
- engine.SetExecutionReady(coordinator.Active)
- engine.SetAuthorize(func(ctx context.Context, command backuptypes.DurableRunCommand) error {
- if cfg.AgentMode && command.UserID == "agent" && command.RequestedWithKey == "" {
+func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {
+ engine := backup.NewEngine(admission, imageService)
+ engine.SetAuthorize(func(ctx context.Context, requester backuptypes.Requester) error {
+ if cfg.AgentMode && requester.UserID == "agent" && requester.APIKeyID == "" {
return nil
}
- if command.UserID == "" {
+ if requester.UserID == "" {
return errors.New("requesting user unavailable")
}
- permissions, err := roles.ResolveExecutionPermissions(ctx, command.UserID, command.RequestedWithKey)
+ permissions, err := roles.ResolveExecutionPermissions(ctx, requester.UserID, requester.APIKeyID)
if err != nil {
return err
}
- if !permissions.Allows(command.Permission, command.EnvironmentID) {
+ if !permissions.Allows(requester.Permission, requester.EnvironmentID) {
return errors.New("requesting user no longer has permission for this backup")
}
- if command.Permission == authz.PermSystemBackupsManage && !permissions.IsGlobalAdmin() {
+ if requester.Permission == authz.PermSystemBackupsManage && !permissions.IsGlobalAdmin() {
return errors.New("system backups require a global administrator")
}
return nil
})
lc.Append(fx.Hook{OnStop: engine.Stop})
- return engine, nil
+ return engine
}
func provideActorRuntimeInternal(
@@ -143,6 +134,17 @@ func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVService
return coordinator, nil
}
+// provideWorkflowEngineInternal starts monitoring after the host is ready; its
+// hook follows the runtime's because the engine depends on it.
+func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {
+ engine, err := flow.New(appCtx, runtime, coordinator, activities)
+ if err != nil {
+ return nil, err
+ }
+ lc.Append(fx.Hook{OnStart: engine.Start, OnStop: engine.Stop})
+ return engine, nil
+}
+
func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, error) {
admission := runs.NewAdmission(runtime.Service(), uuid.New().String())
if err := admission.Register(runtime); err != nil {
@@ -273,10 +275,8 @@ func provideProjectServiceInternal(
type updaterServiceParams struct {
fx.In
- Context context.Context
Config *config.Config
- Lifecycle fx.Lifecycle
- ActorRuntime *francis.Runtime
+ Workflows *flow.Engine
Coordinator *runs.Coordinator
Admission *runs.Admission
Roles *role.RoleService
@@ -293,6 +293,64 @@ type updaterServiceParams struct {
Activity *activity.ActivityService
}
+// provideImageUpdateServiceInternal registers the image-check workflow while the host is still unstarted.
+func provideImageUpdateServiceInternal(
+ db *database.DB,
+ settingsService *settings.SettingsService,
+ registryService *registry.ContainerRegistryService,
+ dockerService *docker.DockerClientService,
+ eventService *event.EventService,
+ notificationService *notification.NotificationService,
+ activityService *activity.ActivityService,
+ engine *flow.Engine,
+) (*imageupdate.ImageUpdateService, error) {
+ service := imageupdate.NewImageUpdateService(db, settingsService, registryService, dockerService, eventService, notificationService, activityService)
+ if err := service.RegisterWorkflows(engine); err != nil {
+ return nil, err
+ }
+ return service, nil
+}
+
+// provideImageServiceInternal registers the auto-patch workflow while the host is still unstarted.
+func provideImageServiceInternal(
+ db *database.DB,
+ dockerService *docker.DockerClientService,
+ registryService *registry.ContainerRegistryService,
+ imageUpdateService *imageupdate.ImageUpdateService,
+ vulnerabilityService *vulnerability.VulnerabilityService,
+ eventService *event.EventService,
+ settingsService *settings.SettingsService,
+ activityService *activity.ActivityService,
+ engine *flow.Engine,
+) (*image.ImageService, error) {
+ service := image.NewImageService(db, dockerService, registryService, imageUpdateService, vulnerabilityService, eventService, settingsService, activityService)
+ if err := service.RegisterWorkflows(engine); err != nil {
+ return nil, err
+ }
+ return service, nil
+}
+
+// provideVulnerabilityServiceInternal registers the scheduled scan workflow while the host is still unstarted.
+func provideVulnerabilityServiceInternal(
+ db *database.DB,
+ cfg *config.Config,
+ dockerService *docker.DockerClientService,
+ eventService *event.EventService,
+ settingsService *settings.SettingsService,
+ notificationService *notification.NotificationService,
+ activityService *activity.ActivityService,
+ registryService *registry.ContainerRegistryService,
+ kvService *kv.KVService,
+ httpClient *http.Client,
+ engine *flow.Engine,
+) (*vulnerability.VulnerabilityService, error) {
+ service := vulnerability.NewVulnerabilityService(db, cfg, dockerService, eventService, settingsService, notificationService, activityService, registryService, kvService, httpClient)
+ if err := service.RegisterWorkflows(engine); err != nil {
+ return nil, err
+ }
+ return service, nil
+}
+
func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterService, error) {
service, err := updater.NewUpdaterService(
p.DB,
@@ -314,21 +372,9 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServ
if err != nil {
return nil, err
}
- if registerActorsErr := service.RegisterActors(p.ActorRuntime); registerActorsErr != nil {
- return nil, registerActorsErr
+ if registerWorkflowsErr := service.RegisterWorkflows(p.Workflows); registerWorkflowsErr != nil {
+ return nil, registerWorkflowsErr
}
- p.Lifecycle.Append(
- fx.Hook{
- OnStart: func(
- context.Context,
- ) error {
- return service.Start(
- p.Context,
- )
- },
- OnStop: service.Stop,
- },
- ) //nolint:contextcheck // Workers inherit the application lifetime after startup returns.
return service, nil
}
@@ -410,3 +456,75 @@ func provideFilesystemWatcherJobInternal(
})
return job, nil
}
+
+// provideGitOpsSyncServiceInternal registers the gitops-sync workflow while the host is still unstarted.
+func provideGitOpsSyncServiceInternal(
+ db *database.DB,
+ repoService *gitrepo.GitRepositoryService,
+ projectService *project.ProjectService,
+ swarmService *swarm.SwarmService,
+ eventService *event.EventService,
+ settingsService *settings.SettingsService,
+ engine *flow.Engine,
+) (*gitops.GitOpsSyncService, error) {
+ service := gitops.NewGitOpsSyncService(db, repoService, projectService, swarmService, eventService, settingsService)
+ if err := service.RegisterWorkflows(engine); err != nil {
+ return nil, err
+ }
+ return service, nil
+}
+
+// provideVolumeServiceInternal registers the volume backup workflows while the host is still unstarted.
+func provideVolumeServiceInternal(
+ lc fx.Lifecycle,
+ db *database.DB,
+ dockerService *docker.DockerClientService,
+ eventService *event.EventService,
+ settingsService *settings.SettingsService,
+ containerService *container.ContainerService,
+ imageService *image.ImageService,
+ backupEngine *backup.Engine,
+ s3Destinations *s3domain.S3DestinationService,
+ cfg *config.Config,
+ recoveryKeys *backup.RecoveryKeyStore,
+ engine *flow.Engine,
+) (*volume.VolumeService, error) {
+ service := volume.NewVolumeService(db, dockerService, eventService, settingsService, containerService, imageService, backupEngine, s3Destinations, cfg, recoveryKeys)
+ if err := service.RegisterWorkflows(engine); err != nil {
+ return nil, err
+ }
+ lc.Append(fx.Hook{OnStop: func(ctx context.Context) error { service.CleanupHelperContainers(ctx); return nil }})
+ return service, nil
+}
+
+// provideSystemServiceInternal registers the system backup workflows while the host is still unstarted.
+func provideSystemServiceInternal(
+ db *database.DB,
+ dockerService *docker.DockerClientService,
+ containerService *container.ContainerService,
+ imageUpdateService *imageupdate.ImageUpdateService,
+ volumeService *volume.VolumeService,
+ networkService *network.NetworkService,
+ settingsService *settings.SettingsService,
+ activityService *activity.ActivityService,
+ versionService *version.VersionService,
+ eventService *event.EventService,
+ projectService *project.ProjectService,
+ backupEngine *backup.Engine,
+ s3Destinations *s3domain.S3DestinationService,
+ recoveryKeys *backup.RecoveryKeyStore,
+ actors *francis.Runtime,
+ cfg *config.Config,
+ environments *environment.EnvironmentService,
+ engine *flow.Engine,
+ roles *role.RoleService,
+) (*system.SystemService, error) {
+ service := system.NewSystemService(
+ db, dockerService, containerService, imageUpdateService, volumeService, networkService, settingsService, activityService,
+ versionService, eventService, projectService, backupEngine, s3Destinations, recoveryKeys, actors, cfg,
+ )
+ if err := service.RegisterWorkflows(engine, environments, roles); err != nil {
+ return nil, err
+ }
+ return service, nil
+}
diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.go
index 4bc89e3de..856440a48 100644
--- a/backend/internal/environment/handler.go
+++ b/backend/internal/environment/handler.go
@@ -26,8 +26,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/environment"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/version"
- "github.com/samber/mo"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
"go.getarcane.app/streams/agg"
@@ -568,14 +567,7 @@ func (h *EnvironmentHandler) DeleteEnvironment(ctx context.Context, input *Delet
return nil, huma.Error500InternalServerError("Failed to delete environment: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Environment deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Environment deleted successfully", ""), nil
}
// TestConnection tests connectivity to an environment.
@@ -622,14 +614,7 @@ func (h *EnvironmentHandler) UpdateHeartbeat(ctx context.Context, input *UpdateH
return nil, huma.Error500InternalServerError("Failed to update heartbeat")
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Heartbeat updated successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Heartbeat updated successfully", ""), nil
}
// PairAgent generates or rotates the local agent pairing token.
@@ -670,15 +655,7 @@ func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnv
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Environment synced successfully",
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Environment synced successfully", activityID), nil
}
func (h *EnvironmentHandler) buildUpdateMapInternal(req *environment.Update, isLocalEnv bool) map[string]any {
@@ -808,14 +785,7 @@ func (h *EnvironmentHandler) PairEnvironment(ctx context.Context, input *PairEnv
slog.InfoContext(ctx, "Environment pairing completed", "environmentId", *envID, "environmentName", env.Name)
h.triggerEnvironmentResourceSyncInternal(ctx, *envID, env.Name, "environment pairing")
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Environment pairing completed successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Environment pairing completed successfully", ""), nil
}
// GetDeploymentSnippets returns deployment snippets for an environment.
diff --git a/backend/internal/event/handler.go b/backend/internal/event/handler.go
index 701c58e81..73ff7a464 100644
--- a/backend/internal/event/handler.go
+++ b/backend/internal/event/handler.go
@@ -121,12 +121,5 @@ func (h *EventHandler) DeleteEvent(ctx context.Context, input *DeleteEventInput)
return nil, huma.Error500InternalServerError("Failed to delete event: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Event deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Event deleted successfully", ""), nil
}
diff --git a/backend/internal/federated/handler.go b/backend/internal/federated/handler.go
index fd7de51a7..dfa3981dc 100644
--- a/backend/internal/federated/handler.go
+++ b/backend/internal/federated/handler.go
@@ -122,14 +122,7 @@ func (h *FederatedCredentialHandler) DeleteFederatedCredential(ctx context.Conte
return nil, federatedCredentialManagementErrorInternal(err)
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Federated credential deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Federated credential deleted successfully", ""), nil
}
func writeFederatedTokenExchangeErrorInternal(c *echo.Context, err error) error {
diff --git a/backend/internal/gitops/children/sync/service.go b/backend/internal/gitops/children/sync/service.go
index 17a5593d6..8c3375199 100644
--- a/backend/internal/gitops/children/sync/service.go
+++ b/backend/internal/gitops/children/sync/service.go
@@ -3,6 +3,7 @@
package sync
import (
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -24,6 +25,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/transport"
"go.getarcane.app/acfs"
"go.getarcane.app/kit/pkg"
"gorm.io/gorm"
@@ -39,18 +41,62 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
-// Service runs deployment syncs. Shared sync limits, error events and job
-// registration stay with the parent and arrive as callbacks.
-type Service struct {
- db *database.DB
- repoService *gitrepo.GitRepositoryService
- projectService *projectpkg.ProjectService
- swarmService *swarm.SwarmService
- eventService *event.EventService
- limits func(context.Context, *projectpkg.GitOpsSync) (int, int64, int64)
- logError func(context.Context, *projectpkg.GitOpsSync, user.Actor, string)
- unregisterJob func(context.Context, string)
-}
+// reservedRootEnvFiles are the project-root env files the override merge owns; git never writes them raw.
+var reservedRootEnvFiles = []string{projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName, projects.GlobalEnvFileName}
+
+type (
+ // Service runs deployment syncs. Shared sync limits, error events and job
+ // registration stay with the parent and arrive as callbacks.
+ Service struct {
+ db *database.DB
+ repoService *gitrepo.GitRepositoryService
+ projectService *projectpkg.ProjectService
+ swarmService *swarm.SwarmService
+ eventService *event.EventService
+ limits func(context.Context, *projectpkg.GitOpsSync) (int, int64, int64)
+ logError func(context.Context, *projectpkg.GitOpsSync, user.Actor, string)
+ unregisterJob func(context.Context, string)
+ }
+
+ // syncRevision is the recovery data a run records so a retry replays the same source revision.
+ syncRevision struct {
+ Commit string `json:"commit"`
+ }
+
+ // preparedSyncSource is the cloned and validated source a sync applies.
+ preparedSyncSource struct {
+ repoPath string
+ commitHash string
+ composeContent string
+ envContent *string
+ overrideContent *string
+ overrideFileName string
+ // replay marks a retry of a pinned revision, which redeploys even when the files are unchanged.
+ replay bool
+ }
+
+ // stagedDirectorySync holds the fully prepared directory-sync result before it
+ // is promoted into the live project path.
+ stagedDirectorySync struct {
+ stagePath string
+ // stageLogical is stagePath relative to the projects directory, the root every staging mutation is confined to.
+ stageLogical string
+ projectsDir string
+ composeFileName string
+ project *projectpkg.Project
+ // syncFiles are the repo files written raw; syncedFiles are their paths.
+ syncFiles []projects.SyncFile
+ syncedFiles []string
+ // oldSyncedFiles are the paths the previous sync tracked.
+ oldSyncedFiles []string
+ // gitEnvContent is the repo's project-root .env, routed through the env merge.
+ gitEnvContent *string
+ serviceCount int
+ contentsChanged bool
+ // backupScope names every live path promotion may create, rewrite or delete.
+ backupScope projects.ProjectUpdateBackupScope
+ }
+)
func New(
db *database.DB,
@@ -74,138 +120,176 @@ func New(
}
}
-// Run clones the sync source and applies it to its project or Swarm stack.
+// Run clones the sync source and applies it to its project or Swarm stack. A retry replays the
+// revision its run pinned, and redeploys it unless the record already confirms that revision.
func (s *Service) Run(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, result *gitops.SyncResult) (*gitops.SyncResult, error) {
- source, err := s.prepareSyncSource(ctx, sync, result, actor)
- if source != nil && source.repoPath != "" {
+ pinned := ""
+ if run, ok := jobcontext.Run(ctx); ok {
+ index := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == sync.ID && len(target.RecoveryData) > 0
+ })
+ var revision syncRevision
+ if index >= 0 {
+ if err := json.Unmarshal(run.Outcome.Targets[index].RecoveryData, &revision); err != nil {
+ return result, err
+ }
+ }
+ pinned = revision.Commit
+ }
+ if pinned != "" && kit.FromPtr(sync.LastSyncCommit) == pinned && kit.FromPtr(sync.LastSyncStatus) == "success" {
+ result.Success = true
+ result.Message = "Revision " + pinned + " is already synced"
+ return result, nil
+ }
+
+ source, err := s.prepareSyncSource(ctx, sync, result, actor, pinned)
+ if source != nil {
defer s.repoService.Discard(ctx, source.repoPath)
}
if err != nil {
return result, err
}
- if source == nil {
- return result, errors.New("prepared sync source is missing")
- }
-
- if sync.TargetType == "swarm_stack" {
- return s.performSwarmStackSyncInternal(ctx, sync, sync.ID, actor, result, source)
- }
+ var syncFiles []projects.SyncFile
if sync.SyncDirectory {
- return s.performDirectorySync(ctx, sync, sync.ID, actor, result, source)
- }
-
- return s.performSingleFileSyncInternal(ctx, sync, sync.ID, actor, result, source)
-}
-
-// DirectoryProject resolves the linked project for a directory sync,
-// relinking or recovering it when the stored reference is stale.
-func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
- return s.getDirectorySyncProjectInternal(ctx, sync)
-}
-
-// ReconcileRevision confirms an interrupted run when the record already holds
-// the source revision that run pinned.
-func ReconcileRevision(ctx context.Context, previous scheduler.Run, record *projectpkg.GitOpsSync, outcome scheduler.Outcome) (scheduler.Outcome, error) {
- target, commit, err := recordedRevision(previous, record.ID)
- if err != nil {
- return outcome, err
+ if syncFiles, err = s.walkAndParseSyncDirectory(ctx, sync, source.repoPath); err != nil {
+ return result, s.failSync(ctx, result, sync, actor, "Failed to walk directory", err)
+ }
}
- if commit == "" || kit.FromPtr(record.LastSyncCommit) != commit || kit.FromPtr(record.LastSyncStatus) != "success" {
- return outcome, nil
+ var project *projectpkg.Project
+ var syncedFiles []string
+ changed := false
+ switch {
+ case sync.TargetType == "swarm_stack":
+ if s.swarmService == nil {
+ return result, s.failSync(ctx, result, sync, actor, "Swarm service is unavailable", errors.New("swarm service is unavailable"))
+ }
+ swarmFiles := make([]swarmtypes.SyncFile, len(syncFiles))
+ for i, file := range syncFiles {
+ swarmFiles[i] = swarmtypes.SyncFile{RelativePath: file.RelativePath, Content: file.Content}
+ syncedFiles = append(syncedFiles, file.RelativePath)
+ }
+ // Swarm tasks pull with only the auth embedded in the service spec, so registry auth is always resolved.
+ request := swarmtypes.StackDeployRequest{
+ Name: sync.ProjectName,
+ ComposeContent: source.composeContent,
+ OverrideContent: kit.FromPtr(source.overrideContent),
+ EnvContent: kit.FromPtr(source.envContent),
+ Files: swarmFiles,
+ Prune: true,
+ WithRegistryAuth: true,
+ WorkingDir: filepath.Dir(filepath.Join(source.repoPath, sync.ComposePath)),
+ }
+ if _, deployErr := s.swarmService.DeployStack(ctx, sync.EnvironmentID, request); deployErr != nil {
+ return result, s.failSync(ctx, result, sync, actor, "Failed to deploy swarm stack", deployErr)
+ }
+ result.Message = fmt.Sprintf("Successfully deployed swarm stack %s from %s", sync.ProjectName, sync.ComposePath)
+ case sync.SyncDirectory:
+ if project, syncedFiles, _, changed, err = s.syncProjectDirectory(ctx, sync, syncFiles, actor); err != nil {
+ message := kit.Ternary(errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken), "GitOps project binding broken", "Failed to sync project directory")
+ return result, s.failSync(ctx, result, sync, actor, message, err)
+ }
+ result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)
+ default:
+ if project, changed, err = s.syncComposeFile(ctx, sync, actor, result, source); err != nil {
+ return result, err
+ }
+ result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
}
- target.Status = scheduler.Succeeded
- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
- return outcome, progressErr
+ if syncedFiles == nil {
+ syncedFiles = []string{filepath.Base(sync.ComposePath)}
+ if source.overrideFileName != "" {
+ syncedFiles = append(syncedFiles, source.overrideFileName)
+ }
}
- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
-}
-// recordedRevision returns the target and source revision an earlier run recorded for syncID.
-func recordedRevision(run scheduler.Run, syncID string) (scheduler.TargetOutcome, string, error) {
- idx := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
- return target.ID == syncID && len(target.RecoveryData) > 0
- })
- if idx < 0 {
- return scheduler.TargetOutcome{}, "", nil
- }
- var revision syncRevision
- if err := json.Unmarshal(run.Outcome.Targets[idx].RecoveryData, &revision); err != nil {
- return scheduler.TargetOutcome{}, "", err
+ if project != nil && (changed || source.replay) {
+ if err = s.deployOnce(ctx, sync, project, actor, source.replay); err != nil {
+ // The files reached disk, so the row keeps their revision and list.
+ errMsg := err.Error()
+ result.Message = "Sync wrote files but redeploy failed"
+ result.Error = &errMsg
+ s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, source.commitHash, syncedFiles)
+ s.logError(ctx, sync, actor, errMsg)
+ return result, err
+ }
}
- return run.Outcome.Targets[idx], revision.Commit, nil
-}
-
-// syncRevision is the recovery data a sync run records so a retry replays the same source revision.
-type syncRevision struct {
- Commit string `json:"commit"`
-}
-// preparedSyncSource captures the repository data needed by the sync execution
-// paths after the source repository has been cloned and validated.
-type preparedSyncSource struct {
- repoPath string
- commitHash string
- composeContent string
- envContent *string
- overrideContent *string
- overrideFileName string
+ s.updateSyncStatus(ctx, sync.ID, "success", "", source.commitHash, syncedFiles)
+ result.Success = true
+ targetKind, targetName := "swarm stack", sync.ProjectName
+ if project != nil {
+ targetKind, targetName = "project", project.Name
+ }
+ _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
+ Type: event.EventTypeGitSyncRun,
+ Severity: event.EventSeveritySuccess,
+ Title: kit.Ternary(project == nil, "Git sync completed for stack", "Git sync completed"),
+ Description: fmt.Sprintf("Successfully synced '%s' to %s '%s'", sync.Name, targetKind, targetName),
+ ResourceType: new("git_sync"),
+ ResourceID: new(sync.ID),
+ ResourceName: new(sync.Name),
+ UserID: new(actor.ID),
+ Username: new(actor.Username),
+ EnvironmentID: new(sync.EnvironmentID),
+ })
+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", sync.ID, "target", targetName)
+ return result, nil
}
-// stagedDirectorySync holds the fully prepared directory-sync result before it
-// is promoted into the live project path.
-type stagedDirectorySync struct {
- stagePath string
- // stageLogical is stagePath expressed relative to the projects directory,
- // which is the confinement root every staging mutation runs against.
- stageLogical string
- projectsDir string
- composeFileName string
- project *projectpkg.Project
- // syncFiles are the repo files written raw; syncedFiles are their paths.
- syncFiles []projects.SyncFile
- syncedFiles []string
- // oldSyncedFiles are the paths the previous sync tracked.
- oldSyncedFiles []string
- // gitEnvContent is the repo's project-root .env, routed through the env merge.
- gitEnvContent *string
- serviceCount int
- contentsChanged bool
- // backupScope names every live path promotion may create, rewrite or delete.
- backupScope projects.ProjectUpdateBackupScope
+// deployOnce redeploys the synced project, recording the deploy on the run. The deploy runs lifecycle hooks, so a
+// replay skips a deploy the run confirmed and refuses one that started without a recorded result.
+func (s *Service) deployOnce(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor, replay bool) error {
+ target := scheduler.TargetOutcome{ResourceType: "gitops_deploy", ID: sync.ID + ":deploy", Status: scheduler.Running}
+ previous, _ := jobcontext.Run(ctx)
+ if index := slices.IndexFunc(previous.Outcome.Targets, func(recorded scheduler.TargetOutcome) bool { return recorded.ID == target.ID }); replay && index >= 0 {
+ switch previous.Outcome.Targets[index].Status {
+ case scheduler.Succeeded:
+ return nil
+ case scheduler.Running:
+ return errors.New("the interrupted deploy may already have run its lifecycle hooks; redeploy the project to finish it")
+ default:
+ }
+ }
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return err
+ }
+ deployErr := s.redeployIfRunningAfterSync(ctx, sync, project, actor)
+ // Interrupted mid-deploy, its effects are unknown, so the deploy stays running for the replay to refuse.
+ if ctx.Err() != nil {
+ return cmp.Or(deployErr, ctx.Err())
+ }
+ target.Status = kit.Ternary(deployErr == nil, scheduler.Succeeded, scheduler.Failed)
+ return errors.Join(deployErr, jobcontext.Progress(ctx, target))
}
-// prepareSyncSource clones the source repository, validates that the configured
-// compose file exists, and reads the compose/env inputs for the sync flow.
-func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsSync, result *gitops.SyncResult, actor user.Actor) (*preparedSyncSource, error) {
+// prepareSyncSource clones the source at the pinned revision, or the branch head, records that revision on
+// the run, and reads the compose, env and override inputs.
+func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsSync, result *gitops.SyncResult, actor user.Actor, pinned string) (*preparedSyncSource, error) {
repository := sync.Repository
if repository == nil {
- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Repository not found", "repository not found")
+ return nil, s.failSync(ctx, result, sync, actor, "Repository not found", errors.New("repository not found"))
}
-
authConfig, err := s.repoService.GetAuthConfig(ctx, repository)
if err != nil {
- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())
- }
-
- // A retry replays the revision its interrupted predecessor recorded, which needs full history.
- pinnedCommit := ""
- if previous, ok := jobcontext.Run(ctx); ok {
- if _, pinnedCommit, err = recordedRevision(previous, sync.ID); err != nil {
- return nil, err
- }
+ return nil, s.failSync(ctx, result, sync, actor, "Failed to get authentication config", err)
}
- repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinnedCommit == "", 1, 0))
+ // Replaying a pinned revision needs full history.
+ repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinned == "", 1, 0))
if err != nil {
- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())
+ // Anything but a rejected or missing repository is worth another attempt.
+ if !errors.Is(err, transport.ErrAuthenticationRequired) && !errors.Is(err, transport.ErrAuthorizationFailed) && !errors.Is(err, transport.ErrRepositoryNotFound) {
+ err = common.Classify(common.ErrUnavailable, err)
+ }
+ return nil, s.failSync(ctx, result, sync, actor, "Failed to clone repository", err)
}
- source := &preparedSyncSource{repoPath: repoPath}
+ source := &preparedSyncSource{repoPath: repoPath, replay: pinned != ""}
source.commitHash, err = s.repoService.GetCurrentCommit(ctx, repoPath)
if err != nil {
slog.WarnContext(ctx, "Failed to get commit hash", "error", err)
}
- if pinnedCommit != "" && pinnedCommit != source.commitHash {
+ if pinned != "" && pinned != source.commitHash {
repo, openErr := git.PlainOpen(repoPath)
if openErr != nil {
return source, openErr
@@ -214,10 +298,10 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps
if treeErr != nil {
return source, treeErr
}
- if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinnedCommit)}); checkoutErr != nil {
+ if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinned)}); checkoutErr != nil {
return source, checkoutErr
}
- source.commitHash = pinnedCommit
+ source.commitHash = pinned
}
if source.commitHash == "" {
return source, errors.New("GitOps source revision is unavailable")
@@ -232,31 +316,25 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps
}
if !s.repoService.FileExists(ctx, repoPath, sync.ComposePath) {
- errMsg := "compose file not found: " + sync.ComposePath
- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)
+ return source, s.failSync(ctx, result, sync, actor, "Compose file not found at "+sync.ComposePath, errors.New("compose file not found: "+sync.ComposePath))
}
source.composeContent, err = s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)
if err != nil {
- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())
+ return source, s.failSync(ctx, result, sync, actor, "Failed to read compose file", err)
}
- envPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")
+ composeDir := filepath.Dir(sync.ComposePath)
+ envPath := filepath.Join(composeDir, ".env")
if s.repoService.FileExists(ctx, repoPath, envPath) {
- content, readFileErr := s.repoService.ReadFile(ctx, repoPath, envPath)
- if readFileErr != nil {
- slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", readFileErr)
+ if content, readErr := s.repoService.ReadFile(ctx, repoPath, envPath); readErr != nil {
+ slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", readErr)
} else {
source.envContent = &content
}
}
- // Detect a Docker Compose override file (compose.override.yaml, etc.) sitting
- // beside the compose file in the repo, mirroring `docker compose` behavior.
- // Only auto-load an override when the compose file was referenced by a
- // standard filename; an explicit custom path is the `-f` case, where docker
- // compose does not auto-load overrides.
+ // Like docker compose, only a standard compose filename auto-loads a sibling override; a custom path is the -f case.
if slices.Contains(projects.ComposeFileCandidates(), filepath.Base(sync.ComposePath)) {
- composeDir := filepath.Dir(sync.ComposePath)
overrideName, overrideContent, overrideFound, overrideErr := projects.ResolveComposeOverride(
func(name string) bool {
return s.repoService.FileExists(ctx, repoPath, filepath.Join(composeDir, name))
@@ -265,1022 +343,447 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps
return s.repoService.ReadFile(ctx, repoPath, filepath.Join(composeDir, name))
},
)
- // Fail the sync on a read error instead of degrading: WriteComposeOverrideFile
- // with nil content removes the project's existing override, so a transient
- // read failure here would delete the override and redeploy without it.
+ // Degrading would delete the project's existing override on a transient read failure.
if overrideErr != nil {
- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose override file", overrideErr.Error())
+ return source, s.failSync(ctx, result, sync, actor, "Failed to read compose override file", overrideErr)
}
if overrideFound {
source.overrideContent = &overrideContent
source.overrideFileName = overrideName
}
}
-
return source, nil
}
-// performDirectorySync runs the directory-sync path and only triggers a
-// redeploy when an already running project's synced contents changed.
-func (
- s *Service,
-) performDirectorySync(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id string,
- actor user.Actor,
- result *gitops.SyncResult,
- source *preparedSyncSource,
-) (
- *gitops.SyncResult,
- error,
-) {
- slog.InfoContext(ctx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)
-
- syncFiles, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)
- if err != nil {
- return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())
+// syncComposeFile applies a single-file sync, creating the project on the first sync. It reports whether the
+// project files changed.
+func (s *Service) syncComposeFile(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, result *gitops.SyncResult, source *preparedSyncSource) (*projectpkg.Project, bool, error) {
+ slog.InfoContext(ctx, "Using single file sync mode", "syncId", sync.ID, "composePath", sync.ComposePath)
+ // Files referenced through COMPOSE_FILE or COMPOSE_ENV_FILES never reach a single-file project.
+ if projects.ComposeEnvRequiresDirectorySync(sync.ComposePath, source.overrideFileName, source.envContent) {
+ return nil, false, s.failSync(ctx, result, sync, actor, "COMPOSE_FILE or COMPOSE_ENV_FILES references additional files",
+ errors.New("the synced .env references additional files via COMPOSE_FILE or COMPOSE_ENV_FILES; enable \"Sync entire directory\" for this sync"))
}
- project, syncedFiles, _, contentsChanged, err := s.syncProjectDirectoryInternal(ctx, sync, syncFiles, actor)
- if err != nil {
- if errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) {
- errMsg := err.Error()
- result.Message = "GitOps project binding broken"
- result.Error = new(errMsg)
- return result, err
+ if projectID := kit.FromPtr(sync.ProjectID); projectID != "" {
+ project, found, err := s.projectService.FindProjectByID(ctx, projectID)
+ if err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to load existing project", fmt.Errorf("failed to get project %s: %w", projectID, err))
}
- return result, s.failSync(ctx, id, result, sync, actor, "Failed to sync project directory", err.Error())
- }
-
- if contentsChanged {
- if redeployErr := s.redeployIfRunningAfterSync(ctx, sync, project, actor, "directory"); redeployErr != nil {
- if errors.Is(redeployErr, common.ErrRedeployAfterSyncFailed) {
- s.markSyncRedeployFailedInternal(ctx, sync, id, source.commitHash, syncedFiles, redeployErr, actor, result)
- }
- return result, redeployErr
+ if !found {
+ slog.WarnContext(ctx, "Existing project not found; GitOps project binding is broken", "projectId", projectID, "syncId", sync.ID)
+ bindingErr := common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s", sync.ID, projectID))
+ return nil, false, s.failSync(ctx, result, sync, actor, "GitOps project binding broken", bindingErr)
}
- }
-
- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)
- result.Success = true
- result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)
- s.logSyncSuccess(ctx, sync, project, actor)
- slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)
-
- return result, nil
-}
-
-// singleFileSyncedFilesInternal builds the tracked-file list for a single-file
-// sync: the compose file's base name plus the override file name when one was
-// resolved. Shared by the compose-only and swarm-stack single-file paths so the
-// two never disagree on what a single-file sync tracked.
-func singleFileSyncedFilesInternal(sync *projectpkg.GitOpsSync, source *preparedSyncSource) []string {
- syncedFiles := []string{filepath.Base(sync.ComposePath)}
- if source.overrideFileName != "" {
- syncedFiles = append(syncedFiles, source.overrideFileName)
- }
- return syncedFiles
-}
-
-// performSingleFileSyncInternal preserves the legacy compose-only Git sync behavior.
-func (
- s *Service,
-) performSingleFileSyncInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id string,
- actor user.Actor,
- result *gitops.SyncResult,
- source *preparedSyncSource,
-) (
- *gitops.SyncResult,
- error,
-) {
- slog.InfoContext(ctx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)
-
- // Single-file sync copies only the one compose file (plus .env and a sibling
- // override). If the synced .env references additional files via COMPOSE_FILE
- // or COMPOSE_ENV_FILES, those files never reach the project, so direct the
- // user to directory sync instead of silently deploying an incomplete
- // selection.
- if projects.ComposeEnvRequiresDirectorySync(sync.ComposePath, source.overrideFileName, source.envContent) {
- return result, s.failSync(ctx, id, result, sync, actor,
- "COMPOSE_FILE or COMPOSE_ENV_FILES references additional files",
- "the synced .env references additional files via COMPOSE_FILE or COMPOSE_ENV_FILES; enable \"Sync entire directory\" for this sync")
- }
-
- syncedFiles := singleFileSyncedFilesInternal(sync, source)
-
- project, err := s.getOrCreateProjectInternal(ctx, sync, id, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, result, actor)
- if err != nil {
- if errors.Is(err, common.ErrRedeployAfterSyncFailed) {
- s.markSyncRedeployFailedInternal(ctx, sync, id, source.commitHash, syncedFiles, err, actor, result)
+ _, changed, err := s.projectService.ApplyGitSyncProjectFiles(ctx, project.ID, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, actor)
+ if err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to update project files", err)
}
- return result, err
+ slog.InfoContext(ctx, "Updated project files", "projectName", project.Name, "projectId", project.ID, "changed", changed)
+ return project, changed, nil
}
- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)
- result.Success = true
- result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
- s.logSyncSuccess(ctx, sync, project, actor)
- slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)
-
- return result, nil
-}
-
-// buildSwarmStackDeployRequestInternal assembles the deploy request for a Git Sync that targets
-// a Swarm stack. WithRegistryAuth is always set so the Git Sync path resolves stored registry
-// credentials the same way a direct stack deploy does. Resolution happens per image at deploy
-// time and yields nothing unless a configured container registry matches that image's host, so
-// stacks built only from public images are unaffected.
-func buildSwarmStackDeployRequestInternal(sync *projectpkg.GitOpsSync, source *preparedSyncSource, overrideContent, envContent string, swarmFiles []swarmtypes.SyncFile) swarmtypes.StackDeployRequest {
- return swarmtypes.StackDeployRequest{
- Name: sync.ProjectName,
- ComposeContent: source.composeContent,
- OverrideContent: overrideContent,
- EnvContent: envContent,
- Files: swarmFiles,
- Prune: true,
- WithRegistryAuth: true,
- WorkingDir: filepath.Dir(filepath.Join(source.repoPath, sync.ComposePath)),
+ // The non-suffixing create: a name collision means the binding is broken, never a "-N" duplicate.
+ project, err := s.projectService.CreateProject(ctx, sync.ProjectName, source.composeContent, source.envContent, projecttypes.CreateProjectWorkspaceManifest{}, nil, nil, nil, actor, false)
+ if errors.Is(err, projects.ErrProjectDirExists) {
+ err = common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s cannot create project %q: a directory with that name "+
+ "already exists; refusing to create a duplicate", sync.ID, projects.SanitizeProjectName(sync.ProjectName)))
+ return nil, false, s.failSync(ctx, result, sync, actor, "GitOps project binding broken", err)
}
-}
-
-// performSwarmStackSyncInternal executes a single file sync targeted at a Swarm Stack
-func (
- s *Service,
-) performSwarmStackSyncInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id string,
- actor user.Actor,
- result *gitops.SyncResult,
- source *preparedSyncSource,
-) (
- *gitops.SyncResult,
- error,
-) {
- slog.InfoContext(ctx, "Deploying Swarm Stack from GitOps sync", "syncId", id, "stackName", sync.ProjectName)
-
- if s.swarmService == nil {
- return result, s.failSync(ctx, id, result, sync, actor, "Swarm service is unavailable", "swarm service is unavailable")
- }
-
- envContent := ""
- if source.envContent != nil {
- envContent = *source.envContent
+ if err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to create project", err)
}
-
- overrideContent := ""
- if source.overrideContent != nil {
- overrideContent = *source.overrideContent
+ if err = s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", sync.ID).Update("project_id", project.ID).Error; err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to update sync with project ID", err)
}
-
- var syncFiles []projects.SyncFile
- if sync.SyncDirectory {
- files, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)
- if err != nil {
- return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())
- }
- syncFiles = files
+ if err = s.db.WithContext(ctx).Model(&projectpkg.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", sync.ID).Error; err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to mark project as GitOps-managed", err)
}
-
- swarmFiles := make([]swarmtypes.SyncFile, len(syncFiles))
- syncedFiles := make([]string, 0, len(syncFiles))
- for i, f := range syncFiles {
- swarmFiles[i] = swarmtypes.SyncFile{
- RelativePath: f.RelativePath,
- Content: f.Content,
- }
- syncedFiles = append(syncedFiles, f.RelativePath)
+ if _, _, err = s.projectService.ApplyGitSyncProjectFiles(ctx, project.ID, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, actor); err != nil {
+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to sync project env files", err)
}
-
- req := buildSwarmStackDeployRequestInternal(sync, source, overrideContent, envContent, swarmFiles)
-
- if _, err := s.swarmService.DeployStack(ctx, sync.EnvironmentID, req); err != nil {
- return result, s.failSync(ctx, id, result, sync, actor, "Failed to deploy swarm stack", err.Error())
- }
-
- if len(syncedFiles) == 0 {
- syncedFiles = singleFileSyncedFilesInternal(sync, source)
- }
- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)
- result.Success = true
- result.Message = fmt.Sprintf("Successfully deployed swarm stack %s from %s", sync.ProjectName, sync.ComposePath)
-
- // Log event
- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
- Type: event.EventTypeGitSyncRun,
- Severity: event.EventSeveritySuccess,
- Title: "Git sync completed for stack",
- Description: fmt.Sprintf("Successfully synced '%s' to swarm stack '%s'", sync.Name, sync.ProjectName),
- ResourceType: new("git_sync"),
- ResourceID: new(sync.ID),
- ResourceName: new(sync.Name),
- UserID: new(actor.ID),
- Username: new(actor.Username),
- EnvironmentID: new(sync.EnvironmentID),
- })
-
- slog.InfoContext(ctx, "GitOps swarm stack sync completed", "syncId", id, "stack", sync.ProjectName)
- return result, nil
+ slog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)
+ return project, true, nil
}
-// redeployIfRunningAfterSync redeploys a project when it is already running,
-// or when the sync has RedeployAfterSync enabled (in which case a stopped
-// project is redeployed too, mirroring Portainer's "Re-pull image" stack
-// option). Returns a common.ErrRedeployAfterSyncFailed when the redeploy
-// itself fails; callers surface that on the sync row's LastSyncError.
-//
-// When neither condition holds — project stopped and RedeployAfterSync off —
-// RedeployProject is never called (Arcane does not start a stopped project on
-// its behalf by default), which also means the image pull that RedeployProject
-// performs as a side effect never happens. Left alone, that can leave a
-// stopped container referencing an image tag that was since re-pointed or
-// pruned upstream, with no local copy to fall back on at next start. If the
-// sync opted into PullImageAfterSync, pull the image here instead so it stays
-// available even while the project is stopped.
-func (s *Service) redeployIfRunningAfterSync(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor, syncMode string) error {
+// redeployIfRunningAfterSync redeploys a running project, or any project when RedeployAfterSync is on. A project
+// left stopped still gets its images pulled when PullImageAfterSync is on, so a re-pointed tag stays available.
+func (s *Service) redeployIfRunningAfterSync(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) error {
details, err := s.projectService.GetProjectDetails(ctx, project.ID, projecttypes.DetailsOptions{})
running := err == nil && (details.Status == string(projectpkg.ProjectStatusRunning) || details.Status == string(projectpkg.ProjectStatusPartiallyRunning))
-
- if !running && !sync.RedeployAfterSync {
- return s.pullImageAfterSyncIfConfiguredInternal(ctx, sync, project, actor)
- }
-
- slog.InfoContext(ctx, "Redeploying project due to content change from Git sync", "syncMode", syncMode, "projectName", project.Name, "projectId", project.ID, "wasRunning", running)
- if redeployProjectErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployProjectErr != nil {
- slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "syncMode", syncMode, "error", redeployProjectErr, "projectId", project.ID)
- return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployProjectErr))
- }
- return nil
-}
-
-// pullImageAfterSyncIfConfiguredInternal pulls each service's image for a
-// project that Git sync changed but did not redeploy (because it wasn't
-// running), when the sync has PullImageAfterSync enabled. Best-effort: a pull
-// failure is logged but never fails the sync itself, mirroring how
-// RedeployProject treats its own pre-deploy pull.
-func (s *Service) pullImageAfterSyncIfConfiguredInternal(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) error {
- if !sync.PullImageAfterSync {
- return nil
- }
-
- credentials, cerr := s.projectService.ResolveRegistryCredentials(ctx)
- if cerr != nil {
- slog.WarnContext(ctx, "failed to resolve registry credentials for post-sync pull", "error", cerr, "projectId", project.ID)
- }
- slog.InfoContext(ctx, "Pulling project images after Git sync (project not running)", "projectName", project.Name, "projectId", project.ID)
- if err := s.projectService.PullProjectImages(ctx, project.ID, io.Discard, actor, credentials); err != nil {
- slog.ErrorContext(ctx, "failed to pull project images after Git sync", "error", err, "projectId", project.ID)
- return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("post-sync image pull failed: %w", err))
+ switch {
+ case running || sync.RedeployAfterSync:
+ slog.InfoContext(ctx, "Redeploying project after Git sync", "projectName", project.Name, "projectId", project.ID, "wasRunning", running)
+ if redeployErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployErr != nil {
+ slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "error", redeployErr, "projectId", project.ID)
+ return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployErr))
+ }
+ case sync.PullImageAfterSync:
+ credentials, credentialsErr := s.projectService.ResolveRegistryCredentials(ctx)
+ if credentialsErr != nil {
+ slog.WarnContext(ctx, "failed to resolve registry credentials for post-sync pull", "error", credentialsErr, "projectId", project.ID)
+ }
+ slog.InfoContext(ctx, "Pulling project images after Git sync (project not running)", "projectName", project.Name, "projectId", project.ID)
+ if pullErr := s.projectService.PullProjectImages(ctx, project.ID, io.Discard, actor, credentials); pullErr != nil {
+ slog.ErrorContext(ctx, "failed to pull project images after Git sync", "error", pullErr, "projectId", project.ID)
+ return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("post-sync image pull failed: %w", pullErr))
+ }
}
return nil
}
-// logSyncSuccess records the Git sync completion event once the filesystem and
-// sync-status updates have already succeeded.
-func (s *Service) logSyncSuccess(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) {
- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
- Type: event.EventTypeGitSyncRun,
- Severity: event.EventSeveritySuccess,
- Title: "Git sync completed",
- Description: fmt.Sprintf("Successfully synced '%s' to project '%s'", sync.Name, project.Name),
- ResourceType: new("git_sync"),
- ResourceID: new(sync.ID),
- ResourceName: new(sync.Name),
- UserID: new(actor.ID),
- Username: new(actor.Username),
- EnvironmentID: new(sync.EnvironmentID),
- })
-}
-
-func (s *Service) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string) {
- now := time.Now()
+// updateSyncStatus stores a run's result on the row. An empty commit keeps the recorded one, and nil
+// syncedFiles keeps the tracked list.
+func (s *Service) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string, syncedFiles []string) {
updates := map[string]any{
- "last_sync_at": now,
+ "last_sync_at": time.Now(),
"last_sync_status": status,
+ "last_sync_error": kit.Ternary[any](errorMsg != "", errorMsg, nil),
}
-
- updates["last_sync_error"] = kit.Ternary[any](errorMsg != "", errorMsg, nil)
-
if commitHash != "" {
updates["last_sync_commit"] = commitHash
}
-
+ if syncedFiles != nil {
+ updates["synced_files"] = projectpkg.EncodeSyncedFiles(syncedFiles)
+ }
if err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(updates).Error; err != nil {
slog.ErrorContext(ctx, "Failed to update sync status", "error", err, "syncId", id)
}
}
-func (s *Service) failSync(ctx context.Context, id string, result *gitops.SyncResult, sync *projectpkg.GitOpsSync, actor user.Actor, message, errMsg string) error {
- result.Message = message
- result.Error = new(errMsg)
- s.updateSyncStatus(ctx, id, "failed", errMsg, "")
- s.logError(ctx, sync, actor, errMsg)
- return fmt.Errorf("%s", errMsg)
-}
-
-// markSyncRedeployFailedInternal records a sync where the file sync wrote
-// cleanly but the auto-redeploy that follows failed (typically a pre-deploy
-// lifecycle hook returning non-zero). The synced-files list and commit hash
-// are preserved on the row so operators can see what reached disk; the
-// error message surfaces the redeploy failure on LastSyncError.
-func (
- s *Service,
-) markSyncRedeployFailedInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id, commitHash string,
- syncedFiles []string,
- redeployErr error,
- actor user.Actor,
- result *gitops.SyncResult,
-) {
- errMsg := redeployErr.Error()
- result.Success = false
- result.Message = "Sync wrote files but redeploy failed"
- result.Error = new(errMsg)
- s.updateSyncStatusWithFiles(ctx, id, "failed", errMsg, commitHash, syncedFiles)
- s.logError(ctx, sync, actor, errMsg)
-}
-
-func (s *Service) disableAutoSyncForBrokenBindingInternal(ctx context.Context, sync *projectpkg.GitOpsSync) {
- if sync == nil || sync.ID == "" {
- return
- }
- if err := s.db.WithContext(ctx).
- Model(&projectpkg.GitOpsSync{}).
- Where("id = ?", sync.ID).
- Update("auto_sync", false).Error; err != nil {
- slog.ErrorContext(ctx, "Failed to disable GitOps auto-sync after broken project binding", "syncId", sync.ID, "error", err)
- }
- sync.AutoSync = false
- s.unregisterJob(ctx, sync.ID)
-}
-
-func (
- s *Service,
-) failSyncAndDisableAutoSyncInternal(
- ctx context.Context,
- id string,
- result *gitops.SyncResult,
- sync *projectpkg.GitOpsSync,
- actor user.Actor,
- message string,
- failure error,
-) error {
- errMsg := failure.Error()
- _ = s.failSync(ctx, id, result, sync, actor, message, errMsg)
- s.disableAutoSyncForBrokenBindingInternal(ctx, sync)
- return failure
-}
-
-func (s *Service) recordBrokenProjectBindingInternal(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, err error) {
- if !errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) || sync == nil {
- return
- }
+// failSync records a failed sync on the result, the row and an event, and returns err. A broken project
+// binding also turns auto-sync off so the sync stops retrying.
+func (s *Service) failSync(ctx context.Context, result *gitops.SyncResult, sync *projectpkg.GitOpsSync, actor user.Actor, message string, err error) error {
errMsg := err.Error()
- s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, "")
+ result.Message = message
+ result.Error = &errMsg
+ s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, "", nil)
s.logError(ctx, sync, actor, errMsg)
- s.disableAutoSyncForBrokenBindingInternal(ctx, sync)
-}
-
-func (
- s *Service,
-) createProjectForSyncInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id, composeContent string,
- envContent, overrideContent *string,
- overrideFileName string,
- result *gitops.SyncResult,
- actor user.Actor,
-) (
- *projectpkg.Project,
- error,
-) {
- // Use the non-suffixing create: a GitOps sync must never mint a "-N" duplicate.
- // A name collision means a project directory already exists for this name, so the
- // binding is broken — fail loudly and disable auto-sync instead of duplicating.
- project, err := s.projectService.CreateProject(ctx, sync.ProjectName, composeContent, envContent, projecttypes.CreateProjectWorkspaceManifest{}, nil, nil, nil, actor, false)
- if err != nil {
- if errors.Is(err, projects.ErrProjectDirExists) {
- bindingErr := common.Classify(
- common.ErrGitOpsSyncProjectBindingBroken,
- fmt.Errorf(
- "GitOps sync project binding broken: sync %s cannot create project %q: a directory with that name "+
- "already exists; refusing to create a duplicate",
- sync.ID,
- projects.SanitizeProjectName(
- sync.ProjectName,
- ),
- ),
- )
-
- return nil, s.failSyncAndDisableAutoSyncInternal(ctx, id, result, sync, actor, "GitOps project binding broken", bindingErr)
- }
- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to create project", err.Error())
- }
-
- // Update sync with project ID
- if linkSyncProjectErr := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(map[string]any{
- "project_id": project.ID,
- }).Error; linkSyncProjectErr != nil {
- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to update sync with project ID", linkSyncProjectErr.Error())
- }
-
- // Mark project as GitOps-managed
- if markProjectManagedErr := s.db.WithContext(ctx).Model(&projectpkg.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", id).Error; markProjectManagedErr != nil {
- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to mark project as GitOps-managed", markProjectManagedErr.Error())
- }
-
- if _, _, applyGitSyncProjectFilesErr := s.projectService.ApplyGitSyncProjectFiles(
- ctx, project.ID, composeContent, envContent, overrideContent, overrideFileName, actor,
- ); applyGitSyncProjectFilesErr != nil {
- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to sync project env files", applyGitSyncProjectFilesErr.Error())
- }
-
- slog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)
-
- // A freshly created project has no containers yet, so it's always
- // "stopped" from RedeployProject's point of view — mirror
- // updateProjectForSyncInternal's post-sync behavior here instead of
- // silently leaving the new project undeployed with no image pulled.
- if sync.RedeployAfterSync {
- slog.InfoContext(ctx, "Redeploying newly created project from Git sync", "projectName", project.Name, "projectId", project.ID)
- if redeployProjectErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployProjectErr != nil {
- slog.ErrorContext(ctx, "Failed to redeploy newly created project after Git sync", "error", redeployProjectErr, "projectId", project.ID)
- return nil, common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployProjectErr))
- }
- } else if pullImageAfterSyncIfConfiguredErr := s.pullImageAfterSyncIfConfiguredInternal(ctx, sync, project, actor); pullImageAfterSyncIfConfiguredErr != nil {
- return nil, pullImageAfterSyncIfConfiguredErr
- }
-
- return project, nil
-}
-
-func (
- s *Service,
-) getOrCreateProjectInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id, composeContent string,
- envContent, overrideContent *string,
- overrideFileName string,
- result *gitops.SyncResult,
- actor user.Actor,
-) (
- *projectpkg.Project,
- error,
-) {
- var project *projectpkg.Project
-
- if sync.ProjectID != nil && *sync.ProjectID != "" {
- var found bool
- var lookupErr error
- project, found, lookupErr = s.projectService.FindProjectByID(ctx, *sync.ProjectID)
- if lookupErr != nil {
- lookupErr = fmt.Errorf("failed to get project %s: %w", *sync.ProjectID, lookupErr)
- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to load existing project", lookupErr.Error())
- }
- if !found {
- err := common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s", sync.ID, *sync.ProjectID))
-
- slog.WarnContext(ctx, "Existing project not found; GitOps project binding is broken", "projectId", *sync.ProjectID, "syncId", sync.ID)
- return nil, s.failSyncAndDisableAutoSyncInternal(ctx, id, result, sync, actor, "GitOps project binding broken", err)
- }
- }
-
- if project == nil {
- return s.createProjectForSyncInternal(ctx, sync, id, composeContent, envContent, overrideContent, overrideFileName, result, actor)
- }
-
- if err := s.updateProjectForSyncInternal(ctx, sync, id, project, composeContent, envContent, overrideContent, overrideFileName, result, actor); err != nil {
- return nil, err
- }
- return project, nil
-}
-
-func (
- s *Service,
-) updateProjectForSyncInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- id string,
- project *projectpkg.Project,
- composeContent string,
- envContent, overrideContent *string,
- overrideFileName string,
- result *gitops.SyncResult,
- actor user.Actor,
-) error {
- _, changed, err := s.projectService.ApplyGitSyncProjectFiles(
- ctx, project.ID, composeContent, envContent, overrideContent, overrideFileName, actor,
- )
- if err != nil {
- return s.failSync(ctx, id, result, sync, actor, "Failed to update project files", err.Error())
- }
- slog.InfoContext(ctx, "Updated project files", "projectName", project.Name, "projectId", project.ID, "changed", changed)
- if !changed {
- return nil
- }
- return s.redeployIfRunningAfterSync(ctx, sync, project, actor, "single-file")
-}
-
-// walkAndParseSyncDirectory walks the repository directory and returns all files with their contents.
-// Returns the list of SyncFile entries and an error if any; it fails if the compose file is missing.
-func (s *Service) walkAndParseSyncDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, repoPath string) ([]projects.SyncFile, error) {
- slog.InfoContext(ctx, "Starting directory walk", "syncId", sync.ID, "composePath", sync.ComposePath)
-
- // Walk the directory to get all files
- maxFiles, maxTotalSize, maxBinarySize := s.limits(ctx, sync)
-
- walkResult, err := s.repoService.WalkDirectory(ctx, repoPath, sync.ComposePath, maxFiles, maxTotalSize, maxBinarySize)
- if err != nil {
- return nil, fmt.Errorf("failed to walk directory: %w", err)
- }
-
- slog.InfoContext(ctx, "Directory walk complete",
- "syncId", sync.ID,
- "totalFiles", walkResult.TotalFiles,
- "totalSize", walkResult.TotalSize,
- "skippedBinaries", walkResult.SkippedBinaries)
-
- // WalkDirectory roots the walk at filepath.Dir(sync.ComposePath), so the
- // compose file is always emitted at the top level as filepath.Base(sync.ComposePath).
- composeFileName := filepath.Base(sync.ComposePath)
- composeFound := false
-
- // Convert walked files to SyncFile format
- syncFiles := make([]projects.SyncFile, len(walkResult.Files))
- for i, f := range walkResult.Files {
- syncFiles[i] = projects.SyncFile{
- RelativePath: f.RelativePath,
- Content: f.Content,
- Executable: f.Executable,
- }
- if f.RelativePath == composeFileName {
- composeFound = true
- }
- }
-
- if !composeFound {
- return nil, fmt.Errorf("compose file %s not found in walked directory", composeFileName)
- }
-
- return syncFiles, nil
-}
-
-// syncProjectDirectoryInternal runs the new directory-sync path end to end:
-// stage files, validate the staged tree, then create or update the project.
-func (
- s *Service,
-) syncProjectDirectoryInternal(
- ctx context.Context,
- sync *projectpkg.GitOpsSync,
- syncFiles []projects.SyncFile,
- actor user.Actor,
-) (
- *projectpkg.Project,
- []string,
- bool,
- bool,
- error,
-) {
- stage, err := s.stageDirectorySyncInternal(ctx, sync, syncFiles)
- if err != nil {
- s.recordBrokenProjectBindingInternal(ctx, sync, actor, err)
- return nil, nil, false, false, err
- }
- // The stage must go even after ctx was cancelled: acfs refuses work on a
- // cancelled context.
- defer func() {
- if stage.stagePath != "" {
- _ = acfs.RemoveAll(context.WithoutCancel(ctx), stage.projectsDir, stage.stageLogical)
- }
- }()
-
- if stage.project == nil {
- project, createDirectorySyncProjectErr := s.createDirectorySyncProjectInternal(ctx, sync, stage, actor)
- if createDirectorySyncProjectErr != nil {
- // A name-collision on create surfaces as a broken-binding error; disable
- // auto-sync so it cannot keep retrying (no-op for other error kinds).
- s.recordBrokenProjectBindingInternal(ctx, sync, actor, createDirectorySyncProjectErr)
- return nil, nil, false, false, createDirectorySyncProjectErr
+ if errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) {
+ if disableErr := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", sync.ID).Update("auto_sync", false).Error; disableErr != nil {
+ slog.ErrorContext(ctx, "Failed to disable GitOps auto-sync after broken project binding", "syncId", sync.ID, "error", disableErr)
}
- return project, stage.syncedFiles, true, true, nil
- }
-
- project, err := s.updateDirectorySyncProjectInternal(ctx, sync, stage)
- if err != nil {
- return nil, nil, false, false, err
+ sync.AutoSync = false
+ s.unregisterJob(ctx, sync.ID)
}
- return project, stage.syncedFiles, false, stage.contentsChanged, nil
+ return err
}
-// stageDirectorySyncInternal builds a temporary project tree that reflects the exact
-// repo layout after sync, including cleanup of files removed from the repo. For an
-// existing project the tree is sparse: only paths the sync touches are materialized
-// and the rest is linked to the live project, so validation never copies it.
-func (s *Service) stageDirectorySyncInternal(ctx context.Context, sync *projectpkg.GitOpsSync, syncFiles []projects.SyncFile) (stage *stagedDirectorySync, err error) {
+// syncProjectDirectory stages and validates the synced tree, then creates or updates the project. It reports
+// the synced paths, whether it created the project, and whether the contents changed.
+func (s *Service) syncProjectDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, syncFiles []projects.SyncFile, actor user.Actor) (*projectpkg.Project, []string, bool, bool, error) {
projectsDir, err := s.projectService.GetProjectsDirectory(ctx)
if err != nil {
- return nil, fmt.Errorf("failed to get projects directory: %w", err)
+ return nil, nil, false, false, fmt.Errorf("failed to get projects directory: %w", err)
}
-
- // The project-root env files are reserved for the three-file override
- // merge applied below by the Project domain rather than a raw
- // overwrite — a raw .env write would silently wipe edits made in Arcane
- // on every sync.
- filteredSyncFiles, gitEnvContent := partitionReservedRootEnvFiles(ctx, syncFiles)
-
stageLogical, err := acfs.MkdirTemp(ctx, projectsDir, "/", ".gitops-sync-stage-*")
if err != nil {
- return nil, fmt.Errorf("failed to create staging directory: %w", err)
+ return nil, nil, false, false, fmt.Errorf("failed to create staging directory: %w", err)
}
- defer func() {
- if err != nil {
- _ = acfs.RemoveAll(context.WithoutCancel(ctx), projectsDir, stageLogical)
- }
- }()
-
- stage = &stagedDirectorySync{
+ stage := &stagedDirectorySync{
stagePath: filepath.Join(projectsDir, filepath.FromSlash(strings.TrimPrefix(stageLogical, "/"))),
stageLogical: stageLogical,
projectsDir: projectsDir,
composeFileName: filepath.Base(sync.ComposePath),
- syncFiles: filteredSyncFiles,
- syncedFiles: make([]string, len(filteredSyncFiles)),
- gitEnvContent: gitEnvContent,
+ // Older syncs may still track .env, which cleanup would delete from the stage before the env merge reads it.
+ oldSyncedFiles: slices.DeleteFunc(sync.SyncedFileList(), func(path string) bool { return slices.Contains(reservedRootEnvFiles, path) }),
contentsChanged: true,
}
- for i, file := range filteredSyncFiles {
- stage.syncedFiles[i] = file.RelativePath
- }
-
- // Syncs created before this fix may still have .env recorded as a tracked
- // file. Drop reserved root env files here too, or CleanupRemovedFiles would
- // treat .env as removed-by-git and delete the stage .env before
- // ApplyGitSyncEnvToDirectory gets a chance to read it for the merge.
- stage.oldSyncedFiles = filterReservedRootEnvFiles(sync.SyncedFileList())
+ // The stage must go even after ctx was cancelled: acfs refuses work on a cancelled context.
+ defer func() {
+ if stage.stagePath != "" {
+ _ = acfs.RemoveAll(context.WithoutCancel(ctx), projectsDir, stageLogical)
+ }
+ }()
- stage.project, err = s.getDirectorySyncProjectInternal(ctx, sync)
- if err != nil {
- return nil, err
+ // Reserved root env files are Arcane's bookkeeping and only come from the env merge, which takes a committed .env as its git source.
+ for _, file := range syncFiles {
+ switch {
+ case !slices.Contains(reservedRootEnvFiles, file.RelativePath):
+ stage.syncFiles = append(stage.syncFiles, file)
+ stage.syncedFiles = append(stage.syncedFiles, file.RelativePath)
+ case file.RelativePath == projects.EffectiveEnvFileName:
+ stage.gitEnvContent = new(string(file.Content))
+ default:
+ slog.WarnContext(ctx, "dropping reserved Arcane env file from git payload; will be re-derived from override merge", "path", file.RelativePath)
+ }
}
+ if stage.project, err = s.DirectoryProject(ctx, sync); err != nil {
+ return nil, nil, false, false, err
+ }
if stage.project != nil {
staleFiles, staleErr := projects.StaleComposeFiles(ctx, stage.project.Path, stage.composeFileName, stage.syncedFiles)
if staleErr != nil {
- return nil, fmt.Errorf("failed to detect stale compose files: %w", staleErr)
+ return nil, nil, false, false, fmt.Errorf("failed to detect stale compose files: %w", staleErr)
}
stage.backupScope.Paths = slices.Concat(stage.syncedFiles, stage.oldSyncedFiles, staleFiles,
[]string{projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName})
- if linkProjectIntoStageErr := linkProjectIntoStage(stage.project.Path, stage.stagePath, stage.backupScope.Paths); linkProjectIntoStageErr != nil {
- return nil, fmt.Errorf("failed to stage current project files: %w", linkProjectIntoStageErr)
+ if linkErr := linkProjectIntoStage(stage.project.Path, stage.stagePath, stage.backupScope.Paths); linkErr != nil {
+ return nil, nil, false, false, fmt.Errorf("failed to stage current project files: %w", linkErr)
}
- if _, seedStageEnvFromDirErr := seedStageEnvFromDir(ctx, stage.project.Path, projectsDir, stage.stagePath); seedStageEnvFromDirErr != nil {
- return nil, seedStageEnvFromDirErr
+ if _, seedErr := seedStageEnvFromDir(ctx, stage.project.Path, projectsDir, stage.stagePath); seedErr != nil {
+ return nil, nil, false, false, seedErr
}
- stage.contentsChanged, err = projects.DirectorySyncContentsChanged(ctx, stage.project.Path, filteredSyncFiles, stage.oldSyncedFiles, stage.composeFileName)
- if err != nil {
- return nil, fmt.Errorf("failed to compare staged directory changes: %w", err)
+ if stage.contentsChanged, err = projects.DirectorySyncContentsChanged(ctx, stage.project.Path, stage.syncFiles, stage.oldSyncedFiles, stage.composeFileName); err != nil {
+ return nil, nil, false, false, fmt.Errorf("failed to compare staged directory changes: %w", err)
}
- } else if seedStageEnvFromCandidateDirErr := s.seedStageEnvFromCandidateDirInternal(ctx, sync, projectsDir, stage.stagePath); seedStageEnvFromCandidateDirErr != nil {
- return nil, seedStageEnvFromCandidateDirErr
+ } else if seedErr := s.seedStageEnvFromCandidateDir(ctx, sync, projectsDir, stage.stagePath); seedErr != nil {
+ return nil, nil, false, false, seedErr
}
- preEnvContent, postEnvContent, err := s.applyDirectorySyncInternal(ctx, stage.stagePath, stage)
+ preEnvContent, postEnvContent, err := s.applyDirectorySync(ctx, stage.stagePath, stage)
if err != nil {
- return nil, err
+ return nil, nil, false, false, err
}
if stage.project != nil && projects.EnvContentChanged(preEnvContent, postEnvContent) {
stage.contentsChanged = true
}
-
- stage.serviceCount, err = s.projectService.ValidateComposeDirectory(ctx, sync.ProjectName, stage.stagePath, stage.composeFileName)
- if err != nil {
- return nil, fmt.Errorf("invalid compose file: %w", err)
+ if stage.serviceCount, err = s.projectService.ValidateComposeDirectory(ctx, sync.ProjectName, stage.stagePath, stage.composeFileName); err != nil {
+ return nil, nil, false, false, fmt.Errorf("invalid compose file: %w", err)
}
- return stage, nil
-}
-
-// applyDirectorySyncInternal applies the sync to targetPath in the order
-// validation saw it: drop files git no longer tracks and stale compose files,
-// write the repo files, then run the project-root env merge. It returns the
-// effective .env content before and after the merge.
-func (s *Service) applyDirectorySyncInternal(ctx context.Context, targetPath string, stage *stagedDirectorySync) (string, string, error) {
- if len(stage.oldSyncedFiles) > 0 {
- if err := projects.CleanupRemovedFiles(ctx, stage.projectsDir, targetPath, stage.oldSyncedFiles, stage.syncedFiles); err != nil {
- return "", "", fmt.Errorf("failed to clean removed synced files: %w", err)
+ if stage.project == nil {
+ project, createErr := s.createDirectorySyncProject(ctx, sync, stage, actor)
+ if createErr != nil {
+ return nil, nil, false, false, createErr
}
+ return project, stage.syncedFiles, true, true, nil
}
-
- if err := projects.RemoveStaleComposeFiles(ctx, targetPath, stage.composeFileName, stage.syncedFiles); err != nil {
- return "", "", fmt.Errorf("failed to remove stale compose files: %w", err)
- }
-
- // Write the repo files (excluding reserved root env files, handled below)
- // after cleanup so validation sees the final on-disk tree exactly as it
- // will exist in the managed project.
- if _, err := projects.WriteSyncedDirectory(ctx, stage.projectsDir, targetPath, stage.syncFiles); err != nil {
- return "", "", fmt.Errorf("failed to write staged sync files: %w", err)
+ project, err := s.updateDirectorySyncProject(ctx, sync, stage)
+ if err != nil {
+ return nil, nil, false, false, err
}
-
- // Route the project-root .env through the same three-file override merge
- // single-file git sync uses: git is source-of-truth, edits made in Arcane
- // become an override that wins, and new git-introduced keys still flow in.
- return s.projectService.ApplyGitSyncEnvToDirectory(ctx, targetPath, stage.projectsDir, stage.gitEnvContent)
+ return project, stage.syncedFiles, false, stage.contentsChanged, nil
}
-// seedStageEnvFromCandidateDirInternal copies env files from a pre-existing project
-// directory at the conventional path (projectsDir/<sanitized-sync-name>/) into the
-// staging directory before initial-sync validation. This lets users pre-seed
-// ${VAR} substitutions via a server-side .env when the compose file in git
-// expects values that the git repo intentionally does not provide. Only env
-// files are touched — other files would conflict with what WriteSyncedDirectory
-// is about to lay down from git.
-func (s *Service) seedStageEnvFromCandidateDirInternal(ctx context.Context, sync *projectpkg.GitOpsSync, projectsDir, stagePath string) error {
+// seedStageEnvFromCandidateDir copies the env files of a pre-existing directory at the conventional project
+// path into a first-sync stage, so a server-side .env can supply ${VAR} values the repo leaves out.
+func (s *Service) seedStageEnvFromCandidateDir(ctx context.Context, sync *projectpkg.GitOpsSync, projectsDir, stagePath string) error {
candidatePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))
info, err := os.Stat(candidatePath)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
if err != nil {
- if errors.Is(err, os.ErrNotExist) {
- return nil
- }
return fmt.Errorf("inspect pre-existing project directory %s: %w", candidatePath, err)
}
if !info.IsDir() {
return nil
}
-
state, err := seedStageEnvFromDir(ctx, candidatePath, projectsDir, stagePath)
- if err != nil {
+ if err != nil || state.HasEffective || (!state.HasGitSource && !state.HasOverride) {
return err
}
- if !state.HasEffective && !state.HasGitSource && !state.HasOverride {
- return nil
+ // Only .env.git and/or project.env exist, so derive .env from them.
+ merged, err := projects.BuildEffectiveEnvContent(state.GitContent, state.OverrideContent)
+ if err != nil {
+ return fmt.Errorf("build effective env from pre-existing project: %w", err)
}
-
- if !state.HasEffective {
- // Only .env.git and/or project.env exist, so derive .env from them.
- merged, mergeErr := projects.BuildEffectiveEnvContent(state.GitContent, state.OverrideContent)
- if mergeErr != nil {
- return fmt.Errorf("build effective env from pre-existing project: %w", mergeErr)
- }
- if writeProjectFileErr := projects.WriteProjectFile(ctx, projectsDir, stagePath, projects.EffectiveEnvFileName, merged); writeProjectFileErr != nil {
- return fmt.Errorf("seed stage .env: %w", writeProjectFileErr)
- }
+ if writeErr := projects.WriteProjectFile(ctx, projectsDir, stagePath, projects.EffectiveEnvFileName, merged); writeErr != nil {
+ return fmt.Errorf("seed stage .env: %w", writeErr)
}
-
- slog.DebugContext(ctx, "Seeded GitOps stage with pre-existing project env files",
- "candidatePath", candidatePath,
- "hasEffective", state.HasEffective,
- "hasGit", state.HasGitSource,
- "hasOverride", state.HasOverride,
- )
return nil
}
-func (s *Service) lookupProjectByPathInternal(ctx context.Context, projectPath string) (*projectpkg.Project, bool, error) {
- var project projectpkg.Project
- if err := s.db.WithContext(ctx).Where("path = ?", projectPath).First(&project).Error; err != nil {
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, false, nil
+// DirectoryProject resolves the linked project for a directory sync, relinking or recovering it when the
+// stored reference is stale. A sync whose project is gone and cannot be recovered has a broken binding.
+func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
+ projectID := kit.FromPtr(sync.ProjectID)
+ if projectID != "" {
+ project, found, err := s.projectService.FindProjectByID(ctx, projectID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get project %s: %w", projectID, err)
}
- return nil, false, fmt.Errorf("failed to get project by path %s: %w", projectPath, err)
+ if found {
+ if rootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); rootErr != nil {
+ return nil, rootErr
+ }
+ return project, s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project)
+ }
+ slog.WarnContext(ctx, "Existing project not found, attempting recovery", "projectId", projectID, "syncId", sync.ID)
}
- return &project, true, nil
+ project, err := s.recoverProject(ctx, sync)
+ if projectID != "" && project == nil {
+ cause := cmp.Or(err, errors.New("no unique recovery candidate was found"))
+ return nil, common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s: %w", sync.ID, projectID, cause))
+ }
+ return project, err
}
-func (s *Service) findRecoverableManagedProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
- var managedProjects []projectpkg.Project
- if err := s.db.WithContext(ctx).
- Where("gitops_managed_by = ?", sync.ID).
- Find(&managedProjects).Error; err != nil {
+// recoverProject finds the one project still managed by the sync, or else adopts the one projects-directory
+// entry that matches the sync's name and holds its compose file.
+func (s *Service) recoverProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
+ var managed []projectpkg.Project
+ if err := s.db.WithContext(ctx).Where("gitops_managed_by = ?", sync.ID).Find(&managed).Error; err != nil {
return nil, fmt.Errorf("failed to list GitOps-managed projects for sync %s: %w", sync.ID, err)
}
-
- matches := make([]projectpkg.Project, 0, len(managedProjects))
- for i := range managedProjects {
- project := managedProjects[i]
- if err := s.projectService.EnsureProjectPathUnderRoot(ctx, &project, true); err != nil {
+ var recovered []projectpkg.Project
+ for i := range managed {
+ if err := s.projectService.EnsureProjectPathUnderRoot(ctx, &managed[i], true); err != nil {
return nil, err
}
- if _, err := s.projectService.ResolveProjectComposeFile(ctx, &project); err != nil {
- if errors.Is(err, common.ErrProjectComposeFileNotFound) {
- continue
- }
+ if _, err := s.projectService.ResolveProjectComposeFile(ctx, &managed[i]); err == nil {
+ recovered = append(recovered, managed[i])
+ } else if !errors.Is(err, common.ErrProjectComposeFileNotFound) {
return nil, err
}
- matches = append(matches, project)
}
-
- switch len(matches) {
+ switch len(recovered) {
case 0:
- return nil, nil
case 1:
- return &matches[0], nil
+ return &recovered[0], s.projectService.EnsureGitOpsProjectLinked(ctx, sync, &recovered[0])
default:
return nil, fmt.Errorf("multiple GitOps-managed projects match sync %s; refusing automatic relink", sync.ID)
}
-}
-func (s *Service) findUniqueProjectDirectoryCandidateInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (string, error) {
projectsDir, err := s.projectService.GetProjectsDirectory(ctx)
if err != nil {
- return "", err
+ return nil, err
}
-
entries, err := acfs.List(ctx, projectsDir, "/")
if err != nil {
- return "", fmt.Errorf("failed to list projects directory %s: %w", projectsDir, err)
+ return nil, fmt.Errorf("failed to list projects directory %s: %w", projectsDir, err)
}
-
composeFileName := strings.TrimSpace(filepath.Base(sync.ComposePath))
if composeFileName == "" || composeFileName == "." {
- return "", nil
+ return nil, nil
}
prefix := projects.SanitizeProjectName(sync.ProjectName)
- matches := make([]string, 0, 1)
+ var matches []string
for _, entry := range entries {
- // Adoption never follows symlinks, and acfs reports a symlink as a
- // non-directory, so a plain directory is the only candidate kind.
- if !entry.IsDirectory {
+ // acfs reports a symlink as a non-directory, so adoption never follows one; Arcane's own scratch dirs never qualify.
+ if !entry.IsDirectory || projects.IsInternalScratchDirName(entry.Name) || (prefix != "" && entry.Name != prefix && !strings.HasPrefix(entry.Name, prefix+"-")) {
continue
}
- // Never adopt one of Arcane's own scratch dirs as a recovery candidate.
- if projects.IsInternalScratchDirName(entry.Name) {
- continue
- }
- if prefix != "" && entry.Name != prefix && !strings.HasPrefix(entry.Name, prefix+"-") {
- continue
- }
-
candidatePath := filepath.Join(projectsDir, entry.Name)
- composePath := filepath.Join(candidatePath, composeFileName)
- if composeEntry, statErr := acfs.Stat(ctx, projectsDir, path.Join(entry.Path, composeFileName), true); statErr == nil {
- if !composeEntry.IsDirectory {
- matches = append(matches, candidatePath)
- }
- } else if !errors.Is(statErr, fs.ErrNotExist) {
- return "", fmt.Errorf("failed to inspect recovery candidate %s: %w", composePath, statErr)
+ composeEntry, statErr := acfs.Stat(ctx, projectsDir, path.Join(entry.Path, composeFileName), true)
+ switch {
+ case statErr == nil && !composeEntry.IsDirectory:
+ matches = append(matches, candidatePath)
+ case statErr != nil && !errors.Is(statErr, fs.ErrNotExist):
+ return nil, fmt.Errorf("failed to inspect recovery candidate %s: %w", filepath.Join(candidatePath, composeFileName), statErr)
}
}
-
switch len(matches) {
case 0:
- return "", nil
+ return nil, nil
case 1:
- return matches[0], nil
default:
- return "", fmt.Errorf("multiple candidate project directories match sync %s; refusing automatic relink", sync.ID)
+ return nil, fmt.Errorf("multiple candidate project directories match sync %s; refusing automatic relink", sync.ID)
}
-}
-func (s *Service) createRecoveredProjectFromDirectoryInternal(ctx context.Context, sync *projectpkg.GitOpsSync, projectPath string) (*projectpkg.Project, error) {
- project := &projectpkg.Project{
+ var project projectpkg.Project
+ err = s.db.WithContext(ctx).Where("path = ?", matches[0]).First(&project).Error
+ if err == nil {
+ if ensureErr := s.projectService.EnsureProjectPathUnderRoot(ctx, &project, true); ensureErr != nil {
+ return nil, ensureErr
+ }
+ return &project, s.projectService.EnsureGitOpsProjectLinked(ctx, sync, &project)
+ }
+ if !errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, fmt.Errorf("failed to get project by path %s: %w", matches[0], err)
+ }
+
+ project = projectpkg.Project{
Name: sync.ProjectName,
- DirName: new(filepath.Base(projectPath)),
- Path: projectPath,
+ DirName: new(filepath.Base(matches[0])),
+ Path: matches[0],
Status: projectpkg.ProjectStatusUnknown,
StatusReason: new("Project recovered from existing GitOps-managed directory"),
- ServiceCount: 0,
- RunningCount: 0,
GitOpsManagedBy: &sync.ID,
}
-
- if serviceCount, err := s.projectService.CountServicesFromCompose(ctx, *project); err == nil {
- project.ServiceCount = serviceCount
- } else {
- slog.WarnContext(ctx, "Failed to count services while recovering GitOps project", "syncId", sync.ID, "path", projectPath, "error", err)
+ if project.ServiceCount, err = s.projectService.CountServicesFromCompose(ctx, project); err != nil {
+ slog.WarnContext(ctx, "Failed to count services while recovering GitOps project", "syncId", sync.ID, "path", matches[0], "error", err)
}
-
- if err := s.projectService.CreateGitOpsManagedProject(ctx, sync, project, user.Actor{}, false); err != nil {
- return nil, err
+ if createErr := s.projectService.CreateGitOpsManagedProject(ctx, sync, &project, user.Actor{}, false); createErr != nil {
+ return nil, createErr
}
-
- return project, nil
+ return &project, nil
}
-func (s *Service) recoverProjectFromDirectoryCandidateInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
- projectPath, err := s.findUniqueProjectDirectoryCandidateInternal(ctx, sync)
- if err != nil || projectPath == "" {
- return nil, err
- }
+// walkAndParseSyncDirectory walks the repository directory and returns all files with their contents.
+// Returns the list of SyncFile entries and an error if any; it fails if the compose file is missing.
+func (s *Service) walkAndParseSyncDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, repoPath string) ([]projects.SyncFile, error) {
+ slog.InfoContext(ctx, "Starting directory walk", "syncId", sync.ID, "composePath", sync.ComposePath)
+
+ // Walk the directory to get all files
+ maxFiles, maxTotalSize, maxBinarySize := s.limits(ctx, sync)
- project, found, err := s.lookupProjectByPathInternal(ctx, projectPath)
+ walkResult, err := s.repoService.WalkDirectory(ctx, repoPath, sync.ComposePath, maxFiles, maxTotalSize, maxBinarySize)
if err != nil {
- return nil, err
- }
- if found {
- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); ensureProjectPathUnderRootErr != nil {
- return nil, ensureProjectPathUnderRootErr
- }
- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); ensureGitOpsProjectLinkedErr != nil {
- return nil, ensureGitOpsProjectLinkedErr
- }
- return project, nil
+ return nil, fmt.Errorf("failed to walk directory: %w", err)
}
- return s.createRecoveredProjectFromDirectoryInternal(ctx, sync, projectPath)
-}
+ slog.InfoContext(ctx, "Directory walk complete",
+ "syncId", sync.ID,
+ "totalFiles", walkResult.TotalFiles,
+ "totalSize", walkResult.TotalSize,
+ "skippedBinaries", walkResult.SkippedBinaries)
-// getDirectorySyncProjectInternal resolves the linked project for a sync when one
-// exists, while tolerating deleted/stale project references.
-func (s *Service) getDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {
- if sync == nil {
- return nil, nil
- }
+ // WalkDirectory roots the walk at filepath.Dir(sync.ComposePath), so the
+ // compose file is always emitted at the top level as filepath.Base(sync.ComposePath).
+ composeFileName := filepath.Base(sync.ComposePath)
+ composeFound := false
- establishedBinding := hasEstablishedProjectBinding(sync)
- if establishedBinding {
- project, found, err := s.projectService.FindProjectByID(ctx, *sync.ProjectID)
- if err != nil {
- return nil, fmt.Errorf("failed to get project %s: %w", *sync.ProjectID, err)
+ // Convert walked files to SyncFile format
+ syncFiles := make([]projects.SyncFile, len(walkResult.Files))
+ for i, f := range walkResult.Files {
+ syncFiles[i] = projects.SyncFile{
+ RelativePath: f.RelativePath,
+ Content: f.Content,
+ Executable: f.Executable,
}
- if found {
- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); ensureProjectPathUnderRootErr != nil {
- return nil, ensureProjectPathUnderRootErr
- }
- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); ensureGitOpsProjectLinkedErr != nil {
- return nil, ensureGitOpsProjectLinkedErr
- }
- return project, nil
+ if f.RelativePath == composeFileName {
+ composeFound = true
}
-
- slog.WarnContext(ctx, "Existing project not found, attempting recovery", "projectId", *sync.ProjectID, "syncId", sync.ID)
}
- project, findRecoverableManagedProjectErr := s.findRecoverableManagedProjectInternal(ctx, sync)
- if findRecoverableManagedProjectErr != nil {
- if establishedBinding {
- return nil, common.Classify(
- common.ErrGitOpsSyncProjectBindingBroken,
- fmt.Errorf(
- "GitOps sync project binding broken: sync %s references missing project %s: %w",
- sync.ID,
- *sync.ProjectID,
- findRecoverableManagedProjectErr,
- ),
- )
- }
- return nil, findRecoverableManagedProjectErr
- }
- if project != nil {
- if err := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); err != nil {
- return nil, err
- }
- return project, nil
+ if !composeFound {
+ return nil, fmt.Errorf("compose file %s not found in walked directory", composeFileName)
}
- project, findRecoverableManagedProjectErr = s.recoverProjectFromDirectoryCandidateInternal(ctx, sync)
- if findRecoverableManagedProjectErr != nil {
- if establishedBinding {
- return nil, common.Classify(
- common.ErrGitOpsSyncProjectBindingBroken,
- fmt.Errorf(
- "GitOps sync project binding broken: sync %s references missing project %s: %w",
- sync.ID,
- *sync.ProjectID,
- findRecoverableManagedProjectErr,
- ),
- )
+ return syncFiles, nil
+}
+
+// applyDirectorySync drops untracked and stale compose files, writes the repo files, then runs the env merge, the
+// order validation saw. It returns the effective .env content before and after the merge.
+func (s *Service) applyDirectorySync(ctx context.Context, targetPath string, stage *stagedDirectorySync) (string, string, error) {
+ if len(stage.oldSyncedFiles) > 0 {
+ if err := projects.CleanupRemovedFiles(ctx, stage.projectsDir, targetPath, stage.oldSyncedFiles, stage.syncedFiles); err != nil {
+ return "", "", fmt.Errorf("failed to clean removed synced files: %w", err)
}
- return nil, findRecoverableManagedProjectErr
}
- if project != nil {
- return project, nil
+
+ if err := projects.RemoveStaleComposeFiles(ctx, targetPath, stage.composeFileName, stage.syncedFiles); err != nil {
+ return "", "", fmt.Errorf("failed to remove stale compose files: %w", err)
}
- if establishedBinding {
- return nil, common.Classify(
- common.ErrGitOpsSyncProjectBindingBroken,
- fmt.Errorf(
- "GitOps sync project binding broken: sync %s references missing project %s: no unique recovery candidate was found",
- sync.ID,
- *sync.ProjectID,
- ),
- )
+ // Write the repo files (excluding reserved root env files, handled below)
+ // after cleanup so validation sees the final on-disk tree exactly as it
+ // will exist in the managed project.
+ if _, err := projects.WriteSyncedDirectory(ctx, stage.projectsDir, targetPath, stage.syncFiles); err != nil {
+ return "", "", fmt.Errorf("failed to write staged sync files: %w", err)
}
- return nil, nil
+ // Route the project-root .env through the same three-file override merge
+ // single-file git sync uses: git is source-of-truth, edits made in Arcane
+ // become an override that wins, and new git-introduced keys still flow in.
+ return s.projectService.ApplyGitSyncEnvToDirectory(ctx, targetPath, stage.projectsDir, stage.gitEnvContent)
}
-// createDirectorySyncProjectInternal promotes a validated staged tree into a new
+// createDirectorySyncProject promotes a validated staged tree into a new
// managed project directory and links it back to the Git sync record.
-func (s *Service) createDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync, actor user.Actor) (*projectpkg.Project, error) {
+func (s *Service) createDirectorySyncProject(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync, actor user.Actor) (*projectpkg.Project, error) {
projectsDir, err := s.projectService.GetProjectsDirectory(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get projects directory: %w", err)
}
// Non-suffixing create: a GitOps sync must never mint a "-N" duplicate. Any
- // adoptable existing directory was already resolved by getDirectorySyncProjectInternal,
+ // adoptable existing directory was already resolved by getDirectorySyncProject,
// so a collision here means the name is taken by an unrelated/unrecoverable dir;
// treat it as a broken binding rather than creating a duplicate.
basePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))
@@ -1337,10 +840,9 @@ func (s *Service) createDirectorySyncProjectInternal(ctx context.Context, sync *
return project, nil
}
-// updateDirectorySyncProjectInternal applies a validated stage to the existing
-// project path in place so running containers keep their bind-mount inodes; a
-// backup scoped to the touched paths allows rollback if promotion fails.
-func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync) (*projectpkg.Project, error) {
+// updateDirectorySyncProject applies a validated stage in place, so running containers keep their bind-mount
+// inodes; a backup scoped to the touched paths rolls a failed promotion back.
+func (s *Service) updateDirectorySyncProject(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync) (*projectpkg.Project, error) {
project := stage.project
projectPath := filepath.Clean(project.Path)
existed := true
@@ -1400,7 +902,7 @@ func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *
return errors.Join(cause, fmt.Errorf("rollback project directory: %w", restoreErr))
}
- if _, _, err := s.applyDirectorySyncInternal(ctx, projectPath, stage); err != nil {
+ if _, _, err := s.applyDirectorySync(ctx, projectPath, stage); err != nil {
return nil, restore(fmt.Errorf("failed to promote staged project directory: %w", err))
}
@@ -1415,30 +917,8 @@ func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *
return project, nil
}
-// updateSyncStatusWithFiles updates sync status including the list of synced files
-func (s *Service) updateSyncStatusWithFiles(ctx context.Context, id, status, errorMsg, commitHash string, syncedFiles []string) {
- now := time.Now()
- updates := map[string]any{
- "last_sync_at": now,
- "last_sync_status": status,
- "synced_files": projectpkg.EncodeSyncedFiles(syncedFiles),
- }
-
- updates["last_sync_error"] = kit.Ternary[any](errorMsg != "", errorMsg, nil)
-
- if commitHash != "" {
- updates["last_sync_commit"] = commitHash
- }
-
- if err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(updates).Error; err != nil {
- slog.ErrorContext(ctx, "Failed to update sync status with files", "error", err, "syncId", id)
- }
-}
-
-// seedStageEnvFromDir copies the readable project-root env files from
-// sourceDir into the stage verbatim and returns the env state it read. A
-// permission-locked file is skipped rather than aborting the whole staging
-// attempt, matching how the env merge leaves such files untouched.
+// seedStageEnvFromDir copies sourceDir's readable project-root env files into the stage and returns the state
+// it read; like the env merge, it skips a permission-locked file.
func seedStageEnvFromDir(ctx context.Context, sourceDir, projectsDir, stagePath string) (projects.ProjectEnvState, error) {
state, err := projects.ReadProjectEnvState(sourceDir)
if err != nil {
@@ -1462,74 +942,8 @@ func seedStageEnvFromDir(ctx context.Context, sourceDir, projectsDir, stagePath
return state, nil
}
-// partitionReservedRootEnvFiles splits syncFiles into the set safe to
-// write directly (everything except the reserved root env files) and the
-// git-sourced root .env content, if the repo has one. A committed
-// .env.git/project.env/.env.global at the project root is dropped rather than
-// raw-written: those are Arcane-owned bookkeeping files that must only ever be
-// produced by the override merge, never by an untrusted git payload.
-func partitionReservedRootEnvFiles(ctx context.Context, syncFiles []projects.SyncFile) ([]projects.SyncFile, *string) {
- filtered := make([]projects.SyncFile, 0, len(syncFiles))
- var gitEnvContent *string
- for _, file := range syncFiles {
- if !isReservedRootEnvFile(file.RelativePath) {
- filtered = append(filtered, file)
- continue
- }
- switch file.RelativePath {
- case projects.EffectiveEnvFileName:
- content := string(file.Content)
- gitEnvContent = &content
- case projects.GlobalEnvFileName:
- slog.WarnContext(ctx, "dropping .env.global from git payload; project-root env files are not raw-written from git and .env.global is not project-scoped",
- "path", file.RelativePath,
- )
- default:
- slog.WarnContext(ctx, "dropping reserved Arcane env file from git payload; will be re-derived from override merge",
- "path", file.RelativePath,
- )
- }
- }
- return filtered, gitEnvContent
-}
-
-// isReservedRootEnvFile reports whether relPath is one of the
-// project-root env bookkeeping files the override-merge system owns (.env,
-// .env.git, project.env, .env.global). These are excluded from the raw
-// directory-sync write and routed through ProjectService.ApplyGitSyncEnvToDirectory
-// instead. Exact-matching RelativePath intentionally leaves nested env files
-// (e.g. svc/.env) in the raw write set — only the project root is
-// merge-managed.
-func isReservedRootEnvFile(relPath string) bool {
- switch relPath {
- case projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName, projects.GlobalEnvFileName:
- return true
- default:
- return false
- }
-}
-
-// filterReservedRootEnvFiles drops reserved root env file paths from a
-// tracked synced-file list. Syncs created before the override-merge migration
-// may still have .env recorded from an earlier sync; leaving it in would make
-// CleanupRemovedFiles treat .env as removed-by-git and delete the live-copied
-// stage .env before ApplyGitSyncEnvToDirectory can read it for the merge.
-func filterReservedRootEnvFiles(paths []string) []string {
- filtered := make([]string, 0, len(paths))
- for _, p := range paths {
- if !isReservedRootEnvFile(p) {
- filtered = append(filtered, p)
- }
- }
- return filtered
-}
-
-// linkProjectIntoStage builds the sparse stage view of an existing
-// project: paths in scope are left for the apply step, directories on the way
-// to them are recreated, and everything else is exposed through a symlink to
-// its live location so validation sees the whole tree without copying it.
-// The links are validation-only and never promoted. acfs refuses to create
-// symlinks by design, so this is the one place the stage is built with os.
+// linkProjectIntoStage symlinks every live path outside scopePaths into the stage, so validation sees the whole
+// project without copying it. The links are never promoted; acfs refuses symlinks, so this uses os.
func linkProjectIntoStage(livePath, stagePath string, scopePaths []string) error {
scope := make(map[string]struct{}, len(scopePaths))
ancestors := make(map[string]struct{})
@@ -1570,7 +984,3 @@ func linkProjectIntoStage(livePath, stagePath string, scopePaths []string) error
}
return link("")
}
-
-func hasEstablishedProjectBinding(sync *projectpkg.GitOpsSync) bool {
- return sync != nil && sync.ProjectID != nil && strings.TrimSpace(*sync.ProjectID) != ""
-}
diff --git a/backend/internal/gitops/children/sync/service_test.go b/backend/internal/gitops/children/sync/service_test.go
index cd6a2f9c9..6c8ccf7c6 100644
--- a/backend/internal/gitops/children/sync/service_test.go
+++ b/backend/internal/gitops/children/sync/service_test.go
@@ -4,15 +4,12 @@ import (
"context"
"encoding/json"
"errors"
- "fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
- "github.com/getarcaneapp/arcane/types/v2/gitops"
- "github.com/getarcaneapp/arcane/types/v2/swarm"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
@@ -31,7 +28,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
)
-func setupSyncTestServiceInternal(t *testing.T) (*Service, *database.DB, string) {
+func setupSyncTestService(t *testing.T) (*Service, *database.DB, string) {
t.Helper()
ctx := t.Context()
@@ -61,7 +58,7 @@ func setupSyncTestServiceInternal(t *testing.T) (*Service, *database.DB, string)
return New(arcaneDB, nil, projectService, nil, eventService, limits, logError, unregisterJob), arcaneDB, projectsDir
}
-func writeFileInternal(t *testing.T, rootDir, relativePath string, content []byte) {
+func writeFile(t *testing.T, rootDir, relativePath string, content []byte) {
t.Helper()
targetPath := filepath.Join(rootDir, relativePath)
@@ -71,10 +68,10 @@ func writeFileInternal(t *testing.T, rootDir, relativePath string, content []byt
// TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision verifies a
// directory sync refuses to create a "-N" sibling when its target name is already taken
-// by a non-adoptable directory; instead it errors as a broken binding and disables auto-sync.
+// by a non-adoptable directory; instead it errors as a broken binding.
func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
// Occupy the target name with a dir that is NOT an adoptable GitOps project
// (it has no matching compose file).
@@ -97,20 +94,16 @@ func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision(
{RelativePath: "docker-compose.yaml", Content: []byte("services:\n app:\n image: nginx:alpine\n")},
}
- _, _, _, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ _, _, _, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.ErrorIs(t, err, common.ErrGitOpsSyncProjectBindingBroken)
_, statErr := os.Stat(filepath.Join(projectsDir, "Dozzle-1"))
require.ErrorIs(t, statErr, os.ErrNotExist, "must not mint a -N duplicate")
-
- var got projectpkg.GitOpsSync
- require.NoError(t, db.Where("id = ?", sync.ID).First(&got).Error)
- assert.False(t, got.AutoSync, "auto-sync should be disabled on broken binding")
}
func TestGitOpsSyncService_SyncProjectDirectory_CreatesProjectPreservingRepoLayout(t *testing.T) {
ctx := t.Context()
- svc, db, _ := setupSyncTestServiceInternal(t)
+ svc, db, _ := setupSyncTestService(t)
sync := &projectpkg.GitOpsSync{
ID: "sync-directory-create",
@@ -149,7 +142,7 @@ services:
},
}
- project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ project, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, project)
require.True(t, created)
@@ -189,7 +182,7 @@ services:
func TestGitOpsSyncService_SyncProjectDirectory_UpdatesProjectAndCleansOldSyncedFiles(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "demo-project")
require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "nested"), 0o755))
@@ -254,7 +247,7 @@ services:
keepBefore, err := os.Stat(filepath.Join(projectPath, "keep.txt"))
require.NoError(t, err)
- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, updatedProject)
require.False(t, created)
@@ -293,7 +286,7 @@ services:
// for https://github.com/getarcaneapp/arcane/issues/2476.
func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGitKey(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "demo-project")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -345,7 +338,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGi
},
}
- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, updatedProject)
require.False(t, created)
@@ -386,7 +379,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGi
// of migrated into project.env.
func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstSyncAfterUpgrade(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "demo-project")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -436,7 +429,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstS
},
}
- updatedProject, _, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, _, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, updatedProject)
require.False(t, created)
@@ -457,7 +450,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstS
// .env.git/project.env on disk are still produced solely by the merge.
func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFiles(t *testing.T) {
ctx := t.Context()
- svc, db, _ := setupSyncTestServiceInternal(t)
+ svc, db, _ := setupSyncTestService(t)
sync := &projectpkg.GitOpsSync{
ID: "sync-directory-reserved-env",
@@ -492,7 +485,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFiles
},
}
- project, syncedFiles, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ project, syncedFiles, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, project)
require.True(t, created)
@@ -515,11 +508,11 @@ func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFiles
func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T) {
ctx := t.Context()
- svc, db, _ := setupSyncTestServiceInternal(t)
+ svc, db, _ := setupSyncTestService(t)
svc.repoService = &gitrepo.GitRepositoryService{Client: git.NewClient("")}
repoPath := t.TempDir()
- writeFileInternal(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:
+ writeFile(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:
traefik:
image: traefik:v3.4
volumes:
@@ -527,7 +520,7 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)
- ./logs:/var/log/traefik
- ./config/dynamic_config.yml:/etc/traefik/dynamic_config.yml:ro
`))
- writeFileInternal(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))
+ writeFile(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))
sync := &projectpkg.GitOpsSync{
ID: "sync-directory-real-walk",
@@ -550,7 +543,7 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)
}
assert.Contains(t, composeContent, "./config/dynamic_config.yml")
- project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ project, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, project)
require.True(t, created)
@@ -577,11 +570,11 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)
func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
svc.repoService = &gitrepo.GitRepositoryService{Client: git.NewClient("")}
repoPath := t.TempDir()
- writeFileInternal(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:
+ writeFile(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:
traefik:
image: traefik:v3.4
volumes:
@@ -589,7 +582,7 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t
- ./logs:/var/log/traefik
- ./config/dynamic_config.yml:/etc/traefik/dynamic_config.yml:ro
`))
- writeFileInternal(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))
+ writeFile(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))
projectPath := filepath.Join(projectsDir, "traefik-project")
require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "config", "dynamic_config.yml"), 0o755))
@@ -621,7 +614,7 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t
syncFiles, err := svc.walkAndParseSyncDirectory(ctx, sync, repoPath)
require.NoError(t, err)
- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, updatedProject)
require.False(t, created)
@@ -650,9 +643,9 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t
assert.True(t, dockerArtifactInfo.IsDir())
}
-func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOnUpdateFailure(t *testing.T) {
+func TestGitOpsSyncService_CreateDirectorySyncProject_RollsBackProjectOnUpdateFailure(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
sync := &projectpkg.GitOpsSync{
ID: "sync-directory-tx-rollback",
@@ -687,7 +680,7 @@ func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOn
_ = db.Callback().Update().Remove(callbackName)
}()
- project, err := svc.createDirectorySyncProjectInternal(ctx, sync, stage, user.Actor{})
+ project, err := svc.createDirectorySyncProject(ctx, sync, stage, user.Actor{})
require.Error(t, err)
require.Nil(t, project)
assert.Contains(t, err.Error(), "failed to mark project as GitOps-managed")
@@ -704,21 +697,21 @@ func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOn
require.ErrorIs(t, statErr, os.ErrNotExist)
}
-// TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedChangesOnUpdateFailure
+// TestGitOpsSyncService_UpdateDirectorySyncProject_RollsBackScopedChangesOnUpdateFailure
// forces the project metadata update to fail after the synced files were written
// to the live project, and verifies the scoped rollback restores exactly what the
// sync touched (in place, keeping inodes) while unrelated files are never copied,
// rewritten or pruned.
-func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedChangesOnUpdateFailure(t *testing.T) {
+func TestGitOpsSyncService_UpdateDirectorySyncProject_RollsBackScopedChangesOnUpdateFailure(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "demo-project")
- writeFileInternal(t, projectPath, "docker-compose.yaml", []byte("services:\n app:\n image: nginx:1.26-alpine\n"))
- writeFileInternal(t, projectPath, "old.txt", []byte("remove me\n"))
- writeFileInternal(t, projectPath, "keep.txt", []byte("keep me\n"))
- writeFileInternal(t, projectPath, ".env", []byte("A=1\n"))
- writeFileInternal(t, projectPath, "data/blob.bin", []byte("bind-mount data\n"))
+ writeFile(t, projectPath, "docker-compose.yaml", []byte("services:\n app:\n image: nginx:1.26-alpine\n"))
+ writeFile(t, projectPath, "old.txt", []byte("remove me\n"))
+ writeFile(t, projectPath, "keep.txt", []byte("keep me\n"))
+ writeFile(t, projectPath, ".env", []byte("A=1\n"))
+ writeFile(t, projectPath, "data/blob.bin", []byte("bind-mount data\n"))
// A directory sits where git now ships a file.
require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "config", "dynamic.yml"), 0o755))
@@ -795,7 +788,7 @@ func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedCha
_ = db.Callback().Update().Remove(callbackName)
}()
- updatedProject, _, _, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, _, _, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.Error(t, err)
require.Nil(t, updatedProject)
assert.Contains(t, err.Error(), "forced project update failure")
@@ -846,9 +839,9 @@ func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedCha
}
}
-func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProjectWhenProjectIDStale(t *testing.T) {
+func TestGitOpsSyncService_GetDirectorySyncProject_RelinksManagedProjectWhenProjectIDStale(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "Radarr-3")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -878,7 +871,7 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProject
}
require.NoError(t, db.Create(project).Error)
- recovered, err := svc.getDirectorySyncProjectInternal(ctx, sync)
+ recovered, err := svc.DirectoryProject(ctx, sync)
require.NoError(t, err)
require.NotNil(t, recovered)
assert.Equal(t, project.ID, recovered.ID)
@@ -889,9 +882,9 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProject
assert.Equal(t, project.ID, *storedSync.ProjectID)
}
-func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirectoryCandidate(t *testing.T) {
+func TestGitOpsSyncService_GetDirectorySyncProject_RecoversUniqueDirectoryCandidate(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "Radarr-3")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -910,7 +903,7 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirecto
}
require.NoError(t, db.Create(sync).Error)
- recovered, err := svc.getDirectorySyncProjectInternal(ctx, sync)
+ recovered, err := svc.DirectoryProject(ctx, sync)
require.NoError(t, err)
require.NotNil(t, recovered)
assert.Equal(t, projectPath, recovered.Path)
@@ -928,120 +921,6 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirecto
assert.Equal(t, 1, storedProject.ServiceCount)
}
-// TestBuildSwarmStackDeployRequestInternal guards the Git Sync swarm deploy request.
-// WithRegistryAuth must always be set: swarm tasks pull with only the auth embedded in
-// the service spec, so a request without it makes every private image fail to pull.
-func TestBuildSwarmStackDeployRequestInternal(t *testing.T) {
- t.Parallel()
-
- files := []swarm.SyncFile{{RelativePath: "configs/app.conf", Content: []byte("k=v")}}
-
- cases := []struct {
- name string
- sync *projectpkg.GitOpsSync
- source *preparedSyncSource
- overrideContent string
- envContent string
- files []swarm.SyncFile
- want swarm.StackDeployRequest
- }{
- {
- name: "populates all fields and enables registry auth",
- sync: &projectpkg.GitOpsSync{ProjectName: "descent", ComposePath: "deploy/swarm/compose.yaml"},
- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},
- overrideContent: "override",
- envContent: "A=1",
- files: files,
- want: swarm.StackDeployRequest{
- Name: "descent",
- ComposeContent: "services: {}\n",
- OverrideContent: "override",
- EnvContent: "A=1",
- Files: files,
- Prune: true,
- WithRegistryAuth: true,
- WorkingDir: filepath.Join(string(filepath.Separator), "tmp", "repo", "deploy", "swarm"),
- },
- },
- {
- name: "enables registry auth with empty optional content and a root-level compose path",
- sync: &projectpkg.GitOpsSync{ProjectName: "ptest", ComposePath: "compose.yaml"},
- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},
- want: swarm.StackDeployRequest{
- Name: "ptest",
- ComposeContent: "services: {}\n",
- Prune: true,
- WithRegistryAuth: true,
- WorkingDir: "/tmp/repo",
- },
- },
- {
- name: "passes an empty file list through unchanged",
- sync: &projectpkg.GitOpsSync{ProjectName: "ptest", ComposePath: "stacks/compose.yaml"},
- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},
- // performSwarmStackSyncInternal always passes a non-nil slice; the builder must not reshape it.
- files: []swarm.SyncFile{},
- want: swarm.StackDeployRequest{
- Name: "ptest",
- ComposeContent: "services: {}\n",
- Files: []swarm.SyncFile{},
- Prune: true,
- WithRegistryAuth: true,
- WorkingDir: filepath.Join(string(filepath.Separator), "tmp", "repo", "stacks"),
- },
- },
- }
-
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- got := buildSwarmStackDeployRequestInternal(tc.sync, tc.source, tc.overrideContent, tc.envContent, tc.files)
- assert.Equal(t, tc.want, got)
- })
- }
-}
-
-func TestMarkSyncRedeployFailedInternal_PersistsErrorOnSyncRow(t *testing.T) {
- ctx := t.Context()
- svc, db, _ := setupSyncTestServiceInternal(t)
- // Event logging requires a real event.EventService; the shared setup leaves it
- // nil since most tests don't exercise the event path.
- require.NoError(t, db.AutoMigrate(&event.Event{}))
- svc.eventService = event.NewEventService(db, config.Load(), nil)
-
- sync := &projectpkg.GitOpsSync{
- ID: "sync-1",
- Name: "redeploy-fail",
- EnvironmentID: "0",
- RepositoryID: "repo-1",
- Branch: "main",
- ComposePath: "compose.yml",
- TargetType: "project",
- }
- require.NoError(t, db.WithContext(ctx).Select("*").Omit("Environment", "Repository", "Project").Create(sync).Error)
-
- result := &gitops.SyncResult{Success: true}
- syncedFiles := []string{"compose.yml", "scripts/pre-deploy.sh"}
- hookErr := common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", errors.New("pre-deploy hook failed: exit 1")))
-
- svc.markSyncRedeployFailedInternal(ctx, sync, sync.ID, "abc123", syncedFiles, hookErr, user.Actor{ID: "user", Username: "tester"}, result)
-
- require.False(t, result.Success)
- require.NotNil(t, result.Error)
- require.Contains(t, *result.Error, "pre-deploy hook failed")
-
- var stored projectpkg.GitOpsSync
- require.NoError(t, db.WithContext(ctx).First(&stored, "id = ?", sync.ID).Error)
- require.NotNil(t, stored.LastSyncStatus)
- require.Equal(t, "failed", *stored.LastSyncStatus)
- require.NotNil(t, stored.LastSyncError)
- require.Contains(t, *stored.LastSyncError, "pre-deploy hook failed")
- // The synced-files list should still be populated so operators can see
- // what reached disk before the redeploy died.
- require.NotNil(t, stored.SyncedFiles)
- require.Contains(t, *stored.SyncedFiles, "compose.yml")
- require.Contains(t, *stored.SyncedFiles, "scripts/pre-deploy.sh")
-}
-
// TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData verifies
// that re-syncing an existing GitOps project tolerates a foreign-owned unreadable file
// inside the project directory (e.g. data a container wrote through a relative bind
@@ -1056,10 +935,10 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData
}
ctx := t.Context()
- svc, db, projectsDir := setupSyncTestServiceInternal(t)
+ svc, db, projectsDir := setupSyncTestService(t)
projectPath := filepath.Join(projectsDir, "demo-project")
- writeFileInternal(t, projectPath, "docker-compose.yaml", []byte(`services:
+ writeFile(t, projectPath, "docker-compose.yaml", []byte(`services:
app:
image: nginx:1.26-alpine
`))
@@ -1107,7 +986,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData
},
}
- updatedProject, _, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})
+ updatedProject, _, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})
require.NoError(t, err)
require.NotNil(t, updatedProject)
require.False(t, created)
diff --git a/backend/internal/gitops/handler.go b/backend/internal/gitops/handler.go
index e51da1a17..0c40ac59c 100644
--- a/backend/internal/gitops/handler.go
+++ b/backend/internal/gitops/handler.go
@@ -280,14 +280,7 @@ func (h *GitOpsSyncHandler) DeleteSync(ctx context.Context, input *DeleteGitOpsS
return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to delete GitOps sync")
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Sync deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Sync deleted successfully", ""), nil
}
// PerformSync manually triggers a sync operation.
diff --git a/backend/internal/gitops/helpers.go b/backend/internal/gitops/helpers.go
index ac9878cb2..bbe4bec18 100644
--- a/backend/internal/gitops/helpers.go
+++ b/backend/internal/gitops/helpers.go
@@ -1,143 +1,20 @@
package gitops
import (
- "encoding/json"
"errors"
- "fmt"
"strings"
- "time"
- "github.com/getarcaneapp/arcane/types/v2/base"
- "github.com/getarcaneapp/arcane/types/v2/gitops"
"go.getarcane.app/kit/pkg"
- "gorm.io/gorm"
-
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
)
-// marshalSyncedFiles converts a list of file paths to JSON for storage
-func MarshalSyncedFiles(files []string) *string {
- if len(files) == 0 {
- return nil
- }
- data, err := json.Marshal(files)
- if err != nil {
- return nil
- }
- return new(string(data))
-}
-
-// parseSyncedFiles parses the JSON array of synced file paths from the database
-func ParseSyncedFiles(syncedFilesJSON *string) []string {
- if syncedFilesJSON == nil || *syncedFilesJSON == "" {
- return nil
- }
- var files []string
- if err := json.Unmarshal([]byte(*syncedFilesJSON), &files); err != nil {
- return nil
- }
- return files
-}
-
-func isUniqueViolation(err error) bool {
- if errors.Is(err, gorm.ErrDuplicatedKey) {
- return true
- }
- message := strings.ToLower(err.Error())
- return strings.Contains(message, "unique constraint") || strings.Contains(message, "duplicate key")
-}
-
-func isGitOpsSyncOverdue(sync *project.GitOpsSync) bool {
- if sync.LastSyncAt == nil {
- return true
- }
- interval := max(sync.SyncInterval, 1)
- return time.Now().After(sync.LastSyncAt.Add(time.Duration(interval) * time.Minute))
-}
-
-func megabytesToBytes(value int) int64 {
- return int64(value) * 1024 * 1024
-}
-
func normalizeSyncLimitSetting(value, defaultValue int) int {
return kit.Ternary(value < 0, defaultValue, value)
}
-func nullableUpdateStringValue(p *string) any {
- if p == nil {
- return nil
- }
- trimmed := strings.TrimSpace(*p)
- return kit.Ternary[any](trimmed == "", nil, trimmed)
-}
-
+// nullableTrimmedString trims p and maps an empty or missing value to nil, which stores NULL.
func nullableTrimmedString(p *string) *string {
- if p == nil {
- return nil
- }
- trimmed := strings.TrimSpace(*p)
- if trimmed == "" {
- return nil
- }
- return &trimmed
-}
-
-// normalizeLifecycleNetworkMode returns the canonical network mode
-// string for a user-supplied value. Empty / whitespace / unset all map to
-// "none" so the secure-by-default behaviour is restored when the user clears
-// the field.
-func normalizeLifecycleNetworkMode(p *string) string {
- if p == nil {
- return "none"
- }
- trimmed := strings.TrimSpace(*p)
- return kit.Ternary(trimmed == "", "none", trimmed)
-}
-
-// resolveEffectiveSyncDirectory mirrors the string resolver but for
-// the SyncDirectory bool: a nil update keeps the existing value, a non-nil
-// update overrides it. On create (current=nil) and no update, defaults to
-// false to match the model's create-time default.
-func resolveEffectiveSyncDirectory(current *project.GitOpsSync, update *bool) bool {
- if update != nil {
- return *update
- }
- if current != nil {
- return current.SyncDirectory
- }
- return false
-}
-
-func resolveLifecycleEffectiveTargetType(current *project.GitOpsSync, update *string) string {
- if update != nil && strings.TrimSpace(*update) != "" {
- return strings.TrimSpace(*update)
- }
- if current != nil && strings.TrimSpace(current.TargetType) != "" {
- return strings.TrimSpace(current.TargetType)
- }
- return "project"
-}
-
-func currentString(sync *project.GitOpsSync, accessor func(*project.GitOpsSync) *string) string {
- if sync == nil {
- return ""
- }
- if p := accessor(sync); p != nil {
- return *p
- }
- return ""
-}
-
-// resolveLifecycleEffectiveString computes the value a string field
-// will take after an update: a nil update leaves the existing value, a
-// non-nil update (including empty string, which clears) overrides it. Used
-// to validate the post-update state of a sync record.
-func resolveLifecycleEffectiveString(existing string, update *string) string {
- if update != nil {
- return strings.TrimSpace(*update)
- }
- return strings.TrimSpace(existing)
+ trimmed := strings.TrimSpace(kit.FromPtr(p))
+ return kit.Ternary(trimmed == "", nil, &trimmed)
}
func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error {
@@ -152,14 +29,3 @@ func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error
}
return nil
}
-
-func normalizeSyncMode(mode string) (string, error) {
- switch strings.TrimSpace(mode) {
- case "", gitops.SyncModeDeploy:
- return gitops.SyncModeDeploy, nil
- case gitops.SyncModeBackup:
- return gitops.SyncModeBackup, nil
- default:
- return "", common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: fmt.Errorf("unsupported sync mode %q", mode)})
- }
-}
diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.go
index a30501164..81fbe4d10 100644
--- a/backend/internal/gitops/service.go
+++ b/backend/internal/gitops/service.go
@@ -19,6 +19,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/lifecycle"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/italypaleale/francis/builtin/workflow"
"go.getarcane.app/acfs"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
@@ -37,261 +38,128 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
)
-type GitOpsSyncService struct {
- db *database.DB
- repoService *gitrepo.GitRepositoryService
- projectService *projectpkg.ProjectService
- swarmService *swarm.SwarmService
- eventService *event.EventService
- settingsService *settings.SettingsService
-
- // jobs carries the scheduler and app lifecycle context, injected
- // post-construction via SetScheduler.
- jobs *entityjobs.Registry
-
- backups *backupRuntimeInternal
- sync *gitopssync.Service
- backup *backup.Service
-}
-
const (
defaultGitSyncTimeout = 5 * time.Minute
defaultMaxSyncFiles = 500
defaultMaxSyncTotalSizeMB = 50
defaultMaxSyncBinarySizeMB = 10
- defaultMaxSyncTotalSize = defaultMaxSyncTotalSizeMB * 1024 * 1024
- defaultMaxSyncBinarySize = defaultMaxSyncBinarySizeMB * 1024 * 1024
- gitOpsSyncAdmissionScopeInternal = "gitops-sync"
+ gitOpsSyncAdmissionScope = "gitops-sync"
)
-// lifecycleConfigInputInternal is the slice of CreateSyncRequest /
-// UpdateSyncRequest fields relevant to the pre-deploy lifecycle hook. Both
-// request types collapse into the same shape so a single validator handles
-// both flows.
-type lifecycleConfigInputInternal struct {
- targetType *string
- scriptPath *string
- runnerImage *string
- env *string
- extraMounts *string
- timeoutSec *int
- networkMode *string
- syncDirectory *bool
-}
-
-func (s *GitOpsSyncService) validateLifecycleConfigInternal(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInputInternal) error {
- lifecycleFieldSet := in.scriptPath != nil || in.runnerImage != nil || in.env != nil || in.extraMounts != nil || in.timeoutSec != nil || in.networkMode != nil
- syncDirectoryChanging := in.syncDirectory != nil
- targetTypeChanging := in.targetType != nil && strings.TrimSpace(*in.targetType) != resolveLifecycleEffectiveTargetType(current, nil)
- effectiveScriptPath := resolveLifecycleEffectiveString(currentString(current, func(c *projectpkg.GitOpsSync) *string { return c.PreDeployScriptPath }), in.scriptPath)
+type (
+ GitOpsSyncService struct {
+ db *database.DB
+ repoService *gitrepo.GitRepositoryService
+ projectService *projectpkg.ProjectService
+ swarmService *swarm.SwarmService
+ eventService *event.EventService
+ settingsService *settings.SettingsService
+
+ // jobs carries the scheduler and app lifecycle context, injected after construction by SetScheduler.
+ jobs *entityjobs.Registry
+ engine *flow.Engine
+ syncWorkflow *flow.Workflow
+
+ backups *backupRuntime
+ sync *gitopssync.Service
+ backup *backup.Service
+ }
+
+ // syncTarget is the gitops-sync workflow input: the sync one scheduled job runs.
+ syncTarget struct {
+ EnvironmentID string `json:"environmentId"`
+ SyncID string `json:"syncId"`
+ }
+
+ // lifecycleConfigInput is the pre-deploy hook slice of a create or update request.
+ lifecycleConfigInput struct {
+ targetType *string
+ scriptPath *string
+ runnerImage *string
+ env *string
+ extraMounts *string
+ timeoutSec *int
+ networkMode *string
+ syncDirectory *bool
+ }
+
+ // backupRuntime holds the in-memory coordination for backup syncs.
+ backupRuntime struct {
+ mu sync.Mutex
+ branches map[string]*sync.Mutex
+ timers map[string]*time.Timer
+ debounce time.Duration
+ }
+)
- // If nothing lifecycle-related is being touched and the resulting state
- // has no script configured, there's nothing to validate.
- if !lifecycleFieldSet && !syncDirectoryChanging && !targetTypeChanging {
- return nil
+func (s *GitOpsSyncService) validateLifecycleConfig(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInput) error {
+ var currentScript, currentRunner *string
+ currentTargetType, currentSyncDirectory := "", false
+ if current != nil {
+ currentScript, currentRunner = current.PreDeployScriptPath, current.PreDeployRunnerImage
+ currentTargetType, currentSyncDirectory = strings.TrimSpace(current.TargetType), current.SyncDirectory
}
- if !lifecycleFieldSet && effectiveScriptPath == "" {
- return nil
+ currentTargetType = cmp.Or(currentTargetType, "project")
+ targetType := cmp.Or(strings.TrimSpace(kit.FromPtr(in.targetType)), currentTargetType)
+ scriptPath := strings.TrimSpace(kit.FromPtr(cmp.Or(in.scriptPath, currentScript)))
+ invalid := func(field, message string) error {
+ return common.Classify(common.ErrValidation, &base.FieldError{Field: field, Err: errors.New(message)})
}
- // Kill switch only blocks updates that actively touch lifecycle fields;
- // toggling unrelated fields on an existing config shouldn't be rejected
- // just because the global setting was turned off afterwards.
+ lifecycleFieldSet := in.scriptPath != nil || in.runnerImage != nil || in.env != nil || in.extraMounts != nil || in.timeoutSec != nil || in.networkMode != nil
+ targetTypeChanging := in.targetType != nil && strings.TrimSpace(*in.targetType) != currentTargetType
+ // Nothing lifecycle-related changes, or the result has no script to validate.
+ if !lifecycleFieldSet && (scriptPath == "" || (in.syncDirectory == nil && !targetTypeChanging)) {
+ return nil
+ }
+ // The kill switch only blocks edits to lifecycle fields, not unrelated edits to a sync that already has a hook.
if lifecycleFieldSet && !s.settingsService.GetBoolSetting(ctx, "lifecycleEnabled", false) {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Pre-deploy lifecycle hooks are disabled. An admin must enable lifecycleEnabled in settings before they can be configured.",
- ),
- },
- )
+ return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are disabled. An admin must enable lifecycleEnabled in settings before they can be configured.")
}
- if effectiveScriptPath != "" {
- if err := s.validateLifecycleScriptConfigInternal(ctx, current, in, effectiveScriptPath); err != nil {
- return err
- }
+ // scriptPath is a POSIX repo path, so path.IsAbs behaves the same on Windows.
+ cleanedScript := filepath.ToSlash(filepath.Clean(scriptPath))
+ switch {
+ case scriptPath == "":
+ case targetType == "swarm_stack":
+ return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are only supported for project syncs.")
+ case len(scriptPath) > 256:
+ return invalid("preDeployScriptPath", "Script path must be 256 characters or fewer.")
+ case path.IsAbs(filepath.ToSlash(scriptPath)):
+ return invalid("preDeployScriptPath", "Script path must be relative to the project directory.")
+ case cleanedScript == ".." || strings.HasPrefix(cleanedScript, "../"):
+ return invalid("preDeployScriptPath", "Script path must not escape the project directory.")
+ case strings.TrimSpace(kit.FromPtr(cmp.Or(in.runnerImage, currentRunner))) == "" && strings.TrimSpace(s.settingsService.GetStringSetting(ctx, "lifecycleDefaultRunnerImage", "")) == "":
+ return invalid("preDeployRunnerImage", "Runner image is required when a script path is set.")
+ case !kit.FromPtr(cmp.Or(in.syncDirectory, &currentSyncDirectory)):
+ return invalid("preDeployScriptPath", "Pre-deploy script requires \"Sync entire directory\" so the script is included in the synced files.")
}
if in.timeoutSec != nil {
+ maxTimeoutSec := s.settingsService.GetIntSetting(ctx, "lifecycleMaxTimeoutSec", lifecycle.DefaultMaxTimeoutSec)
if *in.timeoutSec < 1 {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployTimeoutSec",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Timeout must be at least 1 second.",
- ),
- },
- )
+ return invalid("preDeployTimeoutSec", "Timeout must be at least 1 second.")
}
- maxTimeoutSec := s.settingsService.GetIntSetting(ctx, "lifecycleMaxTimeoutSec", lifecycle.DefaultMaxTimeoutSec)
if maxTimeoutSec > 0 && *in.timeoutSec > maxTimeoutSec {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployTimeoutSec",
- //nolint:staticcheck // Preserve the existing error message.
- Err: fmt.Errorf(
- "Timeout %ds exceeds the lifecycleMaxTimeoutSec setting (%ds).",
- *in.timeoutSec,
- maxTimeoutSec,
- ),
- },
- )
+ return invalid("preDeployTimeoutSec", fmt.Sprintf("Timeout %ds exceeds the lifecycleMaxTimeoutSec setting (%ds).", *in.timeoutSec, maxTimeoutSec))
}
}
-
if _, err := projectpkg.ParseEnvText(in.env); err != nil {
- return common.Classify(common.ErrValidation, &base.FieldError{Field: "preDeployEnv", Err: errors.New(err.Error())})
+ return invalid("preDeployEnv", err.Error())
}
if _, err := projectpkg.ParseExtraMountsText(in.extraMounts); err != nil {
- return common.Classify(common.ErrValidation, &base.FieldError{Field: "preDeployExtraMounts", Err: errors.New(err.Error())})
+ return invalid("preDeployExtraMounts", err.Error())
}
-
return nil
}
-func (s *GitOpsSyncService) validateLifecycleScriptConfigInternal(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInputInternal, scriptPath string) error {
- if resolveLifecycleEffectiveTargetType(current, in.targetType) == "swarm_stack" {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Pre-deploy lifecycle hooks are only supported for project syncs.",
- ),
- },
- )
- }
-
- if len(scriptPath) > 256 {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Script path must be 256 characters or fewer.",
- ),
- },
- )
- }
-
- // scriptPath is a POSIX repo path, not a host path; use path.IsAbs so the
- // check behaves the same on Windows-based contributor machines.
- if path.IsAbs(filepath.ToSlash(scriptPath)) {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Script path must be relative to the project directory.",
- ),
- },
- )
- }
- cleaned := filepath.ToSlash(filepath.Clean(scriptPath))
- if cleaned == ".." || strings.HasPrefix(cleaned, "../") {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Script path must not escape the project directory.",
- ),
- },
- )
- }
-
- effectiveRunnerImage := resolveLifecycleEffectiveString(currentString(current, func(c *projectpkg.GitOpsSync) *string { return c.PreDeployRunnerImage }), in.runnerImage)
- defaultRunnerImage := strings.TrimSpace(s.settingsService.GetStringSetting(ctx, "lifecycleDefaultRunnerImage", ""))
- if effectiveRunnerImage == "" && defaultRunnerImage == "" {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployRunnerImage",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Runner image is required when a script path is set.",
- ),
- },
- )
- }
-
- if !resolveEffectiveSyncDirectory(current, in.syncDirectory) {
- return common.Classify(
- common.ErrValidation,
- &base.FieldError{
- Field: "preDeployScriptPath",
- //nolint:staticcheck // Preserve the existing error message.
- Err: errors.New(
- "Pre-deploy script requires \"Sync entire directory\" so the script is included in the synced files.",
- ),
- },
- )
- }
-
- return nil
-}
-
-// applyLifecycleFieldsToSyncInternal copies lifecycle config from a Create
-// request into a new GitOpsSync row before insert. Strings are trimmed;
-// empty strings remain unset (nil pointer) except for fields that have a
-// non-null default at the DB level.
-func applyLifecycleFieldsToSyncInternal(syncRecord *projectpkg.GitOpsSync, in lifecycleConfigInputInternal) {
- syncRecord.PreDeployScriptPath = nullableTrimmedString(in.scriptPath)
- syncRecord.PreDeployRunnerImage = nullableTrimmedString(in.runnerImage)
- syncRecord.PreDeployEnv = nullableTrimmedString(in.env)
- syncRecord.PreDeployExtraMounts = nullableTrimmedString(in.extraMounts)
- if in.timeoutSec != nil {
- syncRecord.PreDeployTimeoutSec = *in.timeoutSec
- } else {
- syncRecord.PreDeployTimeoutSec = lifecycle.DefaultTimeoutSec
- }
- if mode := normalizeLifecycleNetworkMode(in.networkMode); mode != "" {
- syncRecord.PreDeployNetworkMode = mode
- }
-}
-
-// addLifecycleUpdatesInternal appends lifecycle field updates to the GORM
-// updates map. nil pointers leave the field unchanged; non-nil empty strings
-// clear nullable fields to NULL and reset fixed-default fields to their
-// default value.
-func addLifecycleUpdatesInternal(updates map[string]any, in lifecycleConfigInputInternal) {
- if in.scriptPath != nil {
- updates["pre_deploy_script_path"] = nullableUpdateStringValue(in.scriptPath)
- }
- if in.runnerImage != nil {
- updates["pre_deploy_runner_image"] = nullableUpdateStringValue(in.runnerImage)
- }
- if in.env != nil {
- updates["pre_deploy_env"] = nullableUpdateStringValue(in.env)
- }
- if in.extraMounts != nil {
- updates["pre_deploy_extra_mounts"] = nullableUpdateStringValue(in.extraMounts)
- }
- if in.timeoutSec != nil {
- updates["pre_deploy_timeout_sec"] = *in.timeoutSec
- }
- if in.networkMode != nil {
- updates["pre_deploy_network_mode"] = normalizeLifecycleNetworkMode(in.networkMode)
- }
-}
-
func NewGitOpsSyncService(
db *database.DB,
repoService *gitrepo.GitRepositoryService,
@@ -307,107 +175,114 @@ func NewGitOpsSyncService(
swarmService: swarmService,
eventService: eventService,
settingsService: settingsService,
- jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScopeInternal),
- backups: newBackupRuntimeInternal(),
+ jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScope),
+ backups: &backupRuntime{branches: map[string]*sync.Mutex{}, timers: map[string]*time.Timer{}, debounce: backup.DefaultBackupSaveDebounce},
}
- s.sync = gitopssync.New(db, repoService, projectService, swarmService, eventService, s.getEffectiveSyncLimits, s.logSyncError, s.unregisterSyncJobInternal)
- s.backup = backup.New(db, repoService, projectService, eventService, s.getEffectiveSyncLimits, s.logSyncError, s.backups.branchLock)
+ logError := func(ctx context.Context, syncRecord *projectpkg.GitOpsSync, actor user.Actor, errorMsg string) {
+ _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
+ Type: event.EventTypeGitSyncError,
+ Severity: event.EventSeverityError,
+ Title: "Git sync failed",
+ Description: fmt.Sprintf("Failed to sync '%s': %s", syncRecord.Name, errorMsg),
+ ResourceType: new("git_sync"),
+ ResourceID: new(syncRecord.ID),
+ ResourceName: new(syncRecord.Name),
+ UserID: new(actor.ID),
+ Username: new(actor.Username),
+ EnvironmentID: new(syncRecord.EnvironmentID),
+ })
+ }
+ s.sync = gitopssync.New(db, repoService, projectService, swarmService, eventService, s.getEffectiveSyncLimits, logError, s.jobs.Unregister)
+ s.backup = backup.New(db, repoService, projectService, eventService, s.getEffectiveSyncLimits, logError, s.backups.branchLock)
return s
}
-// SetScheduler injects the job scheduler and the app lifecycle context. It must be
-// called during bootstrap (after the service graph is built) before any per-sync
-// jobs are registered. The lifecycle context is used for background sync kicks so
-// they outlive the request/bootstrap goroutine that triggered them.
-func (s *GitOpsSyncService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {
- return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)
+// RegisterWorkflows defines the gitops-sync workflow each scheduled sync runs. A sync step retries transient
+// clone failures; a retry replays the run's pinned revision.
+func (s *GitOpsSyncService) RegisterWorkflows(engine *flow.Engine) error {
+ s.engine = engine
+ syncWorkflow, err := engine.Define(flow.Definition{
+ Name: "gitops-sync",
+ Version: 1,
+ Fingerprint: "f693aa29ca10c373e1b9d39ab25dee6e396ced7cea2bcfef76f41dba571d576a",
+ Concurrency: 16,
+ Timeout: time.Hour,
+ Steps: []workflow.StepSpec{
+ workflow.Step("sync", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var target syncTarget
+ if err := t.Payload(&target); err != nil {
+ return nil, err
+ }
+ // The row is re-read on every run, so a deleted sync unregisters itself and a disabled one skips.
+ syncRecord, err := s.getSyncByID(ctx, target.EnvironmentID, target.SyncID, false)
+ if errors.Is(err, common.ErrNotFound) {
+ slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", target.SyncID)
+ s.jobs.Unregister(ctx, target.SyncID)
+ return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ }
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ if !syncRecord.AutoSync {
+ return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ }
+ if previous, ok := jobcontext.Run(ctx); ok {
+ if outcome := jobcontext.ConfirmedTarget(previous, target.SyncID); outcome.Status == scheduler.Succeeded {
+ return outcome, nil
+ }
+ }
+ if runningErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Running}); runningErr != nil {
+ return nil, runningErr
+ }
+ result, err := s.PerformSync(ctx, target.EnvironmentID, target.SyncID, user.SystemUser)
+ if err != nil {
+ slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", target.SyncID, "error", err)
+ return nil, err
+ }
+ if !result.Success {
+ return scheduler.Outcome{Status: scheduler.Skipped, Message: result.Message}, nil
+ }
+ if succeededErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Succeeded}); succeededErr != nil {
+ return nil, succeededErr
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: result.Message}, nil
+ }), workflow.WithMaxAttempts(3), workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),
+ },
+ })
+ s.syncWorkflow = syncWorkflow
+ return err
}
-// runScheduledSyncInternal is the body of a scheduled sync fire. It re-reads the
-// sync each run so a row toggled to AutoSync=false self-cancels. If the row is gone
-// (e.g. deleted out-of-band via raw SQL), the job unregisters itself instead of
-// firing forever; transient load errors are skipped and retried next tick.
-func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) (scheduler.Outcome, error) {
- syncRecord, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)
- if err != nil {
- if errors.Is(err, common.ErrNotFound) {
- slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", syncID)
- s.unregisterSyncJobInternal(ctx, syncID)
- return scheduler.Outcome{Status: scheduler.Skipped}, nil
- }
- slog.DebugContext(ctx, "gitops auto-sync skipped; failed to load sync", "syncId", syncID, "error", err)
- return scheduler.Outcome{}, err
- }
- if !syncRecord.AutoSync {
- return scheduler.Outcome{Status: scheduler.Skipped}, nil
- }
- if previous, ok := jobcontext.Run(ctx); ok {
- outcome := jobcontext.ConfirmedTarget(previous, syncID)
- if outcome.Status == scheduler.Succeeded {
- return outcome, nil
- }
- }
- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: syncID, Status: scheduler.Running}); progressErr != nil {
- return scheduler.Outcome{}, progressErr
- }
- result, err := s.PerformSync(ctx, environmentID, syncID, user.SystemUser)
- if err != nil {
- slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", syncID, "error", err)
- return scheduler.Outcome{}, err
- }
- if !result.Success {
- return scheduler.Outcome{Status: scheduler.Skipped, Message: result.Message}, nil
- }
- if completeProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: syncID, Status: scheduler.Succeeded}); completeProgressErr != nil {
- return scheduler.Outcome{}, completeProgressErr
- }
- return scheduler.Outcome{Status: scheduler.Succeeded, Message: result.Message}, nil
+// SetScheduler injects the scheduler, the admission gate, and the app lifecycle context background syncs run on.
+// It must run during bootstrap, before any per-sync job is registered.
+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {
+ return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)
}
-// registerSyncJobInternal schedules a single sync on a fixed "@every Nm"
-// interval. The run body re-reads the sync each fire, so a row deleted or
-// toggled to AutoSync=false self-cancels cleanly, and delegates to PerformSync,
-// which owns its own timeout and the per-sync in-flight guard.
-func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID, environmentID string, intervalMinutes int) {
+// registerSyncJob schedules one sync every intervalMinutes as a gitops-sync workflow run.
+func (s *GitOpsSyncService) registerSyncJob(ctx context.Context, syncID, environmentID string, intervalMinutes int) {
schedule := fmt.Sprintf("@every %dm", max(intervalMinutes, 1))
- s.jobs.Register(ctx, syncID,
- func(_ context.Context) string { return schedule },
- func(ctx context.Context) (scheduler.Outcome, error) {
- return s.runScheduledSyncInternal(ctx, environmentID, syncID)
- },
- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
- outcome := jobcontext.ConfirmedTarget(previous, syncID)
- if outcome.Status == scheduler.Succeeded {
- return outcome, nil
- }
- record, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)
- if err != nil {
- return outcome, err
- }
- return gitopssync.ReconcileRevision(ctx, previous, record, outcome)
- },
- )
-}
-
-func (s *GitOpsSyncService) unregisterSyncJobInternal(ctx context.Context, syncID string) {
- s.jobs.Unregister(ctx, syncID)
+ s.jobs.Add(ctx, &flow.Job{
+ Engine: s.engine,
+ Workflow: s.syncWorkflow,
+ JobName: s.jobs.JobName(syncID),
+ Payload: syncTarget{EnvironmentID: environmentID, SyncID: syncID},
+ ScheduleFn: func(context.Context) string { return schedule },
+ })
}
-// kickSyncInternal runs a sync once in the background on the app lifecycle context.
-// Used when auto-sync is freshly enabled or when a sync is overdue at startup, so
-// the first run does not wait a full interval.
-func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID string) {
+// kickSync submits one run of a sync's job now, so it does not wait for its next interval.
+func (s *GitOpsSyncService) kickSync(ctx context.Context, syncID, trigger string) {
if !s.jobs.Enabled() {
return
}
- if _, err := s.jobs.Scheduler().Submit(ctx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "startup"}); err != nil {
- slog.ErrorContext(ctx, "gitops immediate sync admission failed", "syncId", syncID, "error", err)
+ if _, err := s.jobs.Scheduler().Submit(ctx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: trigger}); err != nil {
+ slog.ErrorContext(ctx, "gitops sync admission failed", "syncId", syncID, "trigger", trigger, "error", err)
}
}
-// RegisterAutoSyncJobsOnStartup registers a dynamic job for every auto-sync-enabled
-// sync and kicks an immediate run for any that are overdue. This replaces the old
-// global polling job so existing syncs keep running after upgrade.
+// RegisterAutoSyncJobsOnStartup registers a job for every auto-sync and kicks the overdue ones, and backups
+// with pending changes, right away.
func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {
if !s.jobs.Enabled() {
return
@@ -419,54 +294,33 @@ func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {
slog.ErrorContext(ctx, "Failed to load auto-sync jobs on startup", "error", err)
return
}
- for i := range syncs {
- syncRecord := syncs[i]
- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
- if isGitOpsSyncOverdue(&syncRecord) || (syncRecord.Mode == gitops.SyncModeBackup && syncRecord.BackupPending) {
- s.kickSyncInternal(ctx, syncRecord.ID)
+ for _, syncRecord := range syncs {
+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
+ overdue := syncRecord.LastSyncAt == nil || time.Since(*syncRecord.LastSyncAt) > time.Duration(max(syncRecord.SyncInterval, 1))*time.Minute
+ if overdue || (syncRecord.Mode == gitops.SyncModeBackup && syncRecord.BackupPending) {
+ s.kickSync(ctx, syncRecord.ID, "startup")
}
}
slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))
}
func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {
- if s.settingsService == nil {
- return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize
- }
-
cfg := s.settingsService.GetSettingsOrDefaults(ctx)
maxFiles := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxFiles.Value, defaultMaxSyncFiles, strconv.Atoi), defaultMaxSyncFiles)
maxTotalSizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxTotalSizeMb.Value, defaultMaxSyncTotalSizeMB, strconv.Atoi), defaultMaxSyncTotalSizeMB)
maxBinarySizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxBinarySizeMb.Value, defaultMaxSyncBinarySizeMB, strconv.Atoi), defaultMaxSyncBinarySizeMB)
-
- return maxFiles, megabytesToBytes(maxTotalSizeMB), megabytesToBytes(maxBinarySizeMB)
+ return maxFiles, int64(maxTotalSizeMB) << 20, int64(maxBinarySizeMB) << 20
}
+// getEffectiveSyncLimits returns the sync's own limits, except where an environment override pins the setting.
func (s *GitOpsSyncService) getEffectiveSyncLimits(ctx context.Context, syncRecord *projectpkg.GitOpsSync) (int, int64, int64) {
- environmentMaxFiles, environmentMaxTotalSize, environmentMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
+ maxFiles, maxTotalSize, maxBinarySize := s.getEnvironmentSyncLimits(ctx)
if syncRecord == nil {
- return environmentMaxFiles, environmentMaxTotalSize, environmentMaxBinarySize
+ return maxFiles, maxTotalSize, maxBinarySize
}
-
- maxFiles := syncRecord.MaxSyncFiles
- maxTotalSize := syncRecord.MaxSyncTotalSize
- maxBinarySize := syncRecord.MaxSyncBinarySize
-
- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxFiles") {
- maxFiles = environmentMaxFiles
- }
- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxTotalSizeMb") {
- maxTotalSize = environmentMaxTotalSize
- }
- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxBinarySizeMb") {
- maxBinarySize = environmentMaxBinarySize
- }
-
- return maxFiles, maxTotalSize, maxBinarySize
-}
-
-func (s *GitOpsSyncService) gitSyncLimitEnvOverrideActiveInternal(key string) bool {
- return s.settingsService != nil && s.settingsService.IsEnvOverrideActive(key)
+ return kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxFiles"), maxFiles, syncRecord.MaxSyncFiles),
+ kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxTotalSizeMb"), maxTotalSize, syncRecord.MaxSyncTotalSize),
+ kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxBinarySizeMb"), maxBinarySize, syncRecord.MaxSyncBinarySize)
}
func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]gitops.GitOpsSync, pagination.Response, gitops.SyncCounts, error) {
@@ -488,10 +342,22 @@ func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentID
q = pagination.ApplyFilter(q, "repository_id", params.Filters["repositoryId"])
q = pagination.ApplyFilter(q, "project_id", params.Filters["projectId"])
- counts, err := s.getFilteredSyncCounts(q)
- if err != nil {
+ var total, active, successful, backups int64
+ if err := errors.Join(
+ q.Session(&gorm.Session{}).Count(&total).Error,
+ q.Session(&gorm.Session{}).Where("auto_sync = ?", true).Count(&active).Error,
+ q.Session(&gorm.Session{}).Where("last_sync_status = ?", "success").Count(&successful).Error,
+ q.Session(&gorm.Session{}).Where("mode = ?", gitops.SyncModeBackup).Count(&backups).Error,
+ ); err != nil {
return nil, pagination.Response{}, gitops.SyncCounts{}, fmt.Errorf("failed to get sync counts: %w", err)
}
+ counts := gitops.SyncCounts{
+ TotalSyncs: int(total),
+ ActiveSyncs: int(active),
+ SuccessfulSyncs: int(successful),
+ DeploySyncs: int(total - backups),
+ BackupSyncs: int(backups),
+ }
paginationResp, err := pagination.PaginateAndSortDB(params, q.Preload("Repository").Preload("Project"), &syncs)
if err != nil {
@@ -506,38 +372,8 @@ func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentID
return out, paginationResp, counts, nil
}
-func (s *GitOpsSyncService) getFilteredSyncCounts(query *gorm.DB) (gitops.SyncCounts, error) {
- var totalSyncs int64
- if err := query.Session(&gorm.Session{}).Count(&totalSyncs).Error; err != nil {
- return gitops.SyncCounts{}, err
- }
-
- var activeSyncs int64
- if err := query.Session(&gorm.Session{}).Where("auto_sync = ?", true).Count(&activeSyncs).Error; err != nil {
- return gitops.SyncCounts{}, err
- }
-
- var successfulSyncs int64
- if err := query.Session(&gorm.Session{}).Where("last_sync_status = ?", "success").Count(&successfulSyncs).Error; err != nil {
- return gitops.SyncCounts{}, err
- }
-
- var backupSyncs int64
- if err := query.Session(&gorm.Session{}).Where("mode = ?", gitops.SyncModeBackup).Count(&backupSyncs).Error; err != nil {
- return gitops.SyncCounts{}, err
- }
-
- return gitops.SyncCounts{
- TotalSyncs: int(totalSyncs),
- ActiveSyncs: int(activeSyncs),
- SuccessfulSyncs: int(successfulSyncs),
- DeploySyncs: int(totalSyncs - backupSyncs),
- BackupSyncs: int(backupSyncs),
- }, nil
-}
-
func (s *GitOpsSyncService) GetSyncByID(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
- syncRecord, err := s.getSyncByIDInternal(ctx, environmentID, id, true)
+ syncRecord, err := s.getSyncByID(ctx, environmentID, id, true)
if err != nil {
if errors.Is(err, common.ErrNotFound) {
slog.WarnContext(ctx, "GitOps sync not found", "syncId", id, "environmentId", environmentID)
@@ -549,7 +385,7 @@ func (s *GitOpsSyncService) GetSyncByID(ctx context.Context, environmentID, id s
return syncRecord, nil
}
-func (s *GitOpsSyncService) getSyncByIDInternal(ctx context.Context, environmentID, id string, preloadAssociations bool) (*projectpkg.GitOpsSync, error) {
+func (s *GitOpsSyncService) getSyncByID(ctx context.Context, environmentID, id string, preloadAssociations bool) (*projectpkg.GitOpsSync, error) {
var syncRecord projectpkg.GitOpsSync
q := s.db.WithContext(ctx).Where("id = ?", id)
if preloadAssociations {
@@ -567,21 +403,16 @@ func (s *GitOpsSyncService) getSyncByIDInternal(ctx context.Context, environment
return &syncRecord, nil
}
-func (s *GitOpsSyncService) getSyncRecordByIDInternal(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
- return s.getSyncByIDInternal(ctx, environmentID, id, false)
-}
-
func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string, req gitops.CreateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {
slog.InfoContext(ctx, "Creating GitOps sync", "environmentId", environmentID, "name", req.Name, "repositoryId", req.RepositoryID)
- mode, err := normalizeSyncMode(req.Mode)
- if err != nil {
- return nil, err
- }
- if mode == gitops.SyncModeDeploy && req.HasBackupOptions() {
+ mode := cmp.Or(strings.TrimSpace(req.Mode), gitops.SyncModeDeploy)
+ switch {
+ case mode != gitops.SyncModeDeploy && mode != gitops.SyncModeBackup:
+ return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: fmt.Errorf("unsupported sync mode %q", req.Mode)})
+ case mode == gitops.SyncModeDeploy && req.HasBackupOptions():
return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: errors.New("backup options require mode \"backup\"")})
- }
- if mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ComposePath) == "" {
+ case mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ComposePath) == "":
return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "composePath", Err: errors.New("compose path is required")})
}
@@ -592,61 +423,10 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
}
slog.InfoContext(ctx, "Found repository for GitOps sync", "repositoryId", req.RepositoryID, "repositoryName", repo.Name)
- // Store the project name - use sync name if project name not provided
- projectName := cmp.Or(req.ProjectName, req.Name)
-
- defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
-
- syncRecord := projectpkg.GitOpsSync{
- Name: req.Name,
- EnvironmentID: environmentID,
- RepositoryID: req.RepositoryID,
- Branch: req.Branch,
- ComposePath: req.ComposePath,
- TargetType: req.TargetType,
- ProjectName: projectName,
- ProjectID: nil, // Will be set during first sync
- Mode: mode,
- AutoSync: false,
- SyncInterval: 60,
- SyncDirectory: false, // Default to single-file sync
- MaxSyncFiles: defaultMaxFiles,
- MaxSyncTotalSize: defaultMaxTotalSize,
- MaxSyncBinarySize: defaultMaxBinarySize,
- BackupOnSave: true,
- }
-
- linkProjectID := strings.TrimSpace(req.ProjectID)
-
- if req.AutoSync != nil {
- syncRecord.AutoSync = *req.AutoSync
- }
- if req.SyncInterval != nil {
- syncRecord.SyncInterval = *req.SyncInterval
+ if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {
+ return nil, limitsErr
}
- if req.SyncDirectory != nil {
- syncRecord.SyncDirectory = *req.SyncDirectory
- }
- if req.PullImageAfterSync != nil {
- syncRecord.PullImageAfterSync = *req.PullImageAfterSync
- }
- if req.RedeployAfterSync != nil {
- syncRecord.RedeployAfterSync = *req.RedeployAfterSync
- }
- if validateSyncLimitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); validateSyncLimitsErr != nil {
- return nil, validateSyncLimitsErr
- }
- if req.MaxSyncFiles != nil {
- syncRecord.MaxSyncFiles = *req.MaxSyncFiles
- }
- if req.MaxSyncTotalSize != nil {
- syncRecord.MaxSyncTotalSize = *req.MaxSyncTotalSize
- }
- if req.MaxSyncBinarySize != nil {
- syncRecord.MaxSyncBinarySize = *req.MaxSyncBinarySize
- }
-
- lifecycleCfg := lifecycleConfigInputInternal{
+ lifecycleCfg := lifecycleConfigInput{
targetType: &req.TargetType,
scriptPath: req.PreDeployScriptPath,
runnerImage: req.PreDeployRunnerImage,
@@ -656,25 +436,49 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
networkMode: req.PreDeployNetworkMode,
syncDirectory: req.SyncDirectory,
}
- if validateLifecycleConfigErr := s.validateLifecycleConfigInternal(ctx, nil, lifecycleCfg); validateLifecycleConfigErr != nil {
- return nil, validateLifecycleConfigErr
+ if lifecycleErr := s.validateLifecycleConfig(ctx, nil, lifecycleCfg); lifecycleErr != nil {
+ return nil, lifecycleErr
}
- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleCfg)
- adoptedProject, err := s.insertSyncRecordInternal(ctx, &syncRecord, req, mode, linkProjectID)
+ defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
+ syncRecord := projectpkg.GitOpsSync{
+ Name: req.Name,
+ EnvironmentID: environmentID,
+ RepositoryID: req.RepositoryID,
+ Branch: req.Branch,
+ ComposePath: req.ComposePath,
+ TargetType: req.TargetType,
+ ProjectName: cmp.Or(req.ProjectName, req.Name),
+ Mode: mode,
+ AutoSync: kit.FromPtr(req.AutoSync),
+ SyncInterval: *cmp.Or(req.SyncInterval, new(60)),
+ SyncDirectory: kit.FromPtr(req.SyncDirectory),
+ PullImageAfterSync: kit.FromPtr(req.PullImageAfterSync),
+ RedeployAfterSync: kit.FromPtr(req.RedeployAfterSync),
+ MaxSyncFiles: *cmp.Or(req.MaxSyncFiles, &defaultMaxFiles),
+ MaxSyncTotalSize: *cmp.Or(req.MaxSyncTotalSize, &defaultMaxTotalSize),
+ MaxSyncBinarySize: *cmp.Or(req.MaxSyncBinarySize, &defaultMaxBinarySize),
+ BackupOnSave: true,
+ PreDeployScriptPath: nullableTrimmedString(req.PreDeployScriptPath),
+ PreDeployRunnerImage: nullableTrimmedString(req.PreDeployRunnerImage),
+ PreDeployEnv: nullableTrimmedString(req.PreDeployEnv),
+ PreDeployExtraMounts: nullableTrimmedString(req.PreDeployExtraMounts),
+ PreDeployTimeoutSec: *cmp.Or(req.PreDeployTimeoutSec, new(lifecycle.DefaultTimeoutSec)),
+ PreDeployNetworkMode: cmp.Or(strings.TrimSpace(kit.FromPtr(req.PreDeployNetworkMode)), "none"),
+ }
+
+ adoptedProject, err := s.insertSyncRecord(ctx, &syncRecord, req, mode)
if err != nil {
return nil, err
}
slog.InfoContext(ctx, "GitOps sync created successfully", "syncId", syncRecord.ID, "name", syncRecord.Name)
-
if adoptedProject != nil {
adoptedProject.GitOpsManagedBy = &syncRecord.ID
- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, &syncRecord, adoptedProject); ensureGitOpsProjectLinkedErr != nil {
- return nil, fmt.Errorf("failed to link existing project: %w", ensureGitOpsProjectLinkedErr)
+ if linkErr := s.projectService.EnsureGitOpsProjectLinked(ctx, &syncRecord, adoptedProject); linkErr != nil {
+ return nil, fmt.Errorf("failed to link existing project: %w", linkErr)
}
}
- // Log event
_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeGitSyncCreate,
Severity: event.EventSeveritySuccess,
@@ -688,60 +492,44 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
EnvironmentID: new(syncRecord.EnvironmentID),
})
- if _, performSyncErr := s.PerformSync(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor); performSyncErr != nil {
- slog.ErrorContext(ctx, "Failed to perform initial sync after creation", "syncId", syncRecord.ID, "error", performSyncErr)
- // Don't fail the entire creation - the sync config exists and can be retried
+ // The sync exists even when its first run fails, so that failure can be retried rather than failing creation.
+ if _, syncErr := s.PerformSync(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor); syncErr != nil {
+ slog.ErrorContext(ctx, "Failed to perform initial sync after creation", "syncId", syncRecord.ID, "error", syncErr)
}
-
- // Register the recurring job if auto-sync is on. The initial sync above already
- // ran once, so no extra kick is needed here.
+ // The initial sync above already ran, so the job needs no kick.
if syncRecord.AutoSync {
- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
}
-
return s.GetSyncByID(ctx, "", syncRecord.ID)
}
-// prepareDeployProjectLinkInternal validates that an existing project can be
-// adopted by a deploy sync: it must exist, not be deployed from Git already,
-// and not be backed up to Git.
-func (s *GitOpsSyncService) prepareDeployProjectLinkInternal(ctx context.Context, tx *gorm.DB, req gitops.CreateSyncRequest) (*projectpkg.Project, error) {
- if strings.TrimSpace(req.TargetType) == "swarm_stack" {
- return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "projectId", Err: errors.New("an existing project cannot be linked to a swarm stack sync")})
- }
- project, err := projectpkg.LockProjectForSync(tx, req.ProjectID)
- if err != nil {
- return nil, err
- }
- if project.GitOpsManagedBy != nil && strings.TrimSpace(*project.GitOpsManagedBy) != "" {
- return nil, common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))
- }
- var backups int64
- if countBackupsErr := tx.Model(&projectpkg.GitOpsSync{}).
- Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, project.ID).
- Count(&backups).Error; countBackupsErr != nil {
- return nil, fmt.Errorf("failed to check existing backups: %w", countBackupsErr)
- }
- if backups > 0 {
- return nil, common.Classify(common.ErrConflict, errors.New("project is backed up to Git; disconnect that backup before deploying it from Git"))
- }
- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, false); ensureProjectPathUnderRootErr != nil {
- return nil, ensureProjectPathUnderRootErr
- }
- return project, nil
-}
-
-// insertSyncRecordInternal validates the project link and inserts the sync in
-// one transaction with the project row locked, so a project cannot end up with
-// two Git relationships. It returns the adopted project for deploy links.
-func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRecord *projectpkg.GitOpsSync, req gitops.CreateSyncRequest, mode, linkProjectID string) (*projectpkg.Project, error) {
+// insertSyncRecord inserts the sync in one transaction with any linked project locked, so a project never
+// ends up with two Git relationships. It returns the project a deploy sync adopts.
+func (s *GitOpsSyncService) insertSyncRecord(ctx context.Context, syncRecord *projectpkg.GitOpsSync, req gitops.CreateSyncRequest, mode string) (*projectpkg.Project, error) {
var adoptedProject *projectpkg.Project
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if mode == gitops.SyncModeDeploy && linkProjectID != "" {
- project, err := s.prepareDeployProjectLinkInternal(ctx, tx, req)
+ // An adopted project must exist, and must not be deployed from or backed up to Git already.
+ if mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ProjectID) != "" {
+ if strings.TrimSpace(req.TargetType) == "swarm_stack" {
+ return common.Classify(common.ErrValidation, &base.FieldError{Field: "projectId", Err: errors.New("an existing project cannot be linked to a swarm stack sync")})
+ }
+ project, err := projectpkg.LockProjectForSync(tx, req.ProjectID)
if err != nil {
return err
}
+ if strings.TrimSpace(kit.FromPtr(project.GitOpsManagedBy)) != "" {
+ return common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))
+ }
+ var backups int64
+ if countErr := tx.Model(&projectpkg.GitOpsSync{}).Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, project.ID).Count(&backups).Error; countErr != nil {
+ return fmt.Errorf("failed to check existing backups: %w", countErr)
+ }
+ if backups > 0 {
+ return common.Classify(common.ErrConflict, errors.New("project is backed up to Git; disconnect that backup before deploying it from Git"))
+ }
+ if rootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, false); rootErr != nil {
+ return rootErr
+ }
adoptedProject = project
syncRecord.ProjectID = &project.ID
syncRecord.ProjectName = project.Name
@@ -752,10 +540,10 @@ func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRe
}
}
- // Select("*") forces explicit zero values (e.g. "0 = unlimited" sync limits and
- // unset pre-deploy fields) to persist instead of GORM substituting column defaults.
+ // Select("*") persists explicit zero values, such as unlimited sync limits, instead of column defaults.
if err := tx.Select("*").Omit("Environment", "Repository", "Project").Create(syncRecord).Error; err != nil { //nolint:unqueryvet // intentional Select("*"); see comment above
- if isUniqueViolation(err) {
+ message := strings.ToLower(err.Error())
+ if errors.Is(err, gorm.ErrDuplicatedKey) || strings.Contains(message, "unique constraint") || strings.Contains(message, "duplicate key") {
return common.Classify(common.ErrConflict, errors.New("project already has a Git backup; disconnect it first"))
}
slog.ErrorContext(ctx, "Failed to create GitOps sync in database", "name", req.Name, "repositoryId", req.RepositoryID, "environmentId", syncRecord.EnvironmentID, "error", err)
@@ -775,10 +563,7 @@ func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRe
}
return nil
})
- if err != nil {
- return nil, err
- }
- return adoptedProject, nil
+ return adoptedProject, err
}
func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id string, req gitops.UpdateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {
@@ -787,16 +572,10 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
return nil, err
}
- // Capture state needed to reconcile the dynamic job after the update.
+ // Captured before the update so the job can follow the new state.
oldAutoSync := syncRecord.AutoSync
- newAutoSync := syncRecord.AutoSync
- if req.AutoSync != nil {
- newAutoSync = *req.AutoSync
- }
- newInterval := syncRecord.SyncInterval
- if req.SyncInterval != nil {
- newInterval = *req.SyncInterval
- }
+ newAutoSync := *cmp.Or(req.AutoSync, &syncRecord.AutoSync)
+ newInterval := *cmp.Or(req.SyncInterval, &syncRecord.SyncInterval)
updates := make(map[string]any)
@@ -804,10 +583,8 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
updates["name"] = *req.Name
}
if req.RepositoryID != nil {
- // Validate repository exists
- _, getRepositoryByIDErr := s.repoService.GetRepositoryByID(ctx, *req.RepositoryID)
- if getRepositoryByIDErr != nil {
- return nil, fmt.Errorf("repository not found: %w", getRepositoryByIDErr)
+ if _, repoErr := s.repoService.GetRepositoryByID(ctx, *req.RepositoryID); repoErr != nil {
+ return nil, fmt.Errorf("repository not found: %w", repoErr)
}
updates["repository_id"] = *req.RepositoryID
}
@@ -838,8 +615,8 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
if req.RedeployAfterSync != nil {
updates["redeploy_after_sync"] = *req.RedeployAfterSync
}
- if validateSyncLimitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); validateSyncLimitsErr != nil {
- return nil, validateSyncLimitsErr
+ if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {
+ return nil, limitsErr
}
if req.MaxSyncFiles != nil {
updates["max_sync_files"] = *req.MaxSyncFiles
@@ -851,11 +628,11 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
updates["max_sync_binary_size"] = *req.MaxSyncBinarySize
}
- if applyModeUpdatesErr := s.backup.ApplyModeUpdates(ctx, syncRecord, req, updates); applyModeUpdatesErr != nil {
- return nil, applyModeUpdatesErr
+ if modeErr := s.backup.ApplyModeUpdates(ctx, syncRecord, req, updates); modeErr != nil {
+ return nil, modeErr
}
- lifecycleCfg := lifecycleConfigInputInternal{
+ lifecycleCfg := lifecycleConfigInput{
targetType: req.TargetType,
scriptPath: req.PreDeployScriptPath,
runnerImage: req.PreDeployRunnerImage,
@@ -865,18 +642,32 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
networkMode: req.PreDeployNetworkMode,
syncDirectory: req.SyncDirectory,
}
- if validateLifecycleConfigErr := s.validateLifecycleConfigInternal(ctx, syncRecord, lifecycleCfg); validateLifecycleConfigErr != nil {
- return nil, validateLifecycleConfigErr
+ if lifecycleErr := s.validateLifecycleConfig(ctx, syncRecord, lifecycleCfg); lifecycleErr != nil {
+ return nil, lifecycleErr
+ }
+ // A nil field stays unchanged; an empty one clears the column, or resets the network mode to "none".
+ for column, value := range map[string]*string{
+ "pre_deploy_script_path": lifecycleCfg.scriptPath,
+ "pre_deploy_runner_image": lifecycleCfg.runnerImage,
+ "pre_deploy_env": lifecycleCfg.env,
+ "pre_deploy_extra_mounts": lifecycleCfg.extraMounts,
+ } {
+ if value != nil {
+ updates[column] = nullableTrimmedString(value)
+ }
+ }
+ if lifecycleCfg.timeoutSec != nil {
+ updates["pre_deploy_timeout_sec"] = *lifecycleCfg.timeoutSec
+ }
+ if lifecycleCfg.networkMode != nil {
+ updates["pre_deploy_network_mode"] = cmp.Or(strings.TrimSpace(*lifecycleCfg.networkMode), "none")
}
- addLifecycleUpdatesInternal(updates, lifecycleCfg)
if len(updates) > 0 {
// Loaded associations must not overwrite explicitly updated foreign keys.
- if updateSyncErr := s.db.WithContext(ctx).Model(syncRecord).Omit(clause.Associations).Updates(updates).Error; updateSyncErr != nil {
- return nil, fmt.Errorf("failed to update sync: %w", updateSyncErr)
+ if updateErr := s.db.WithContext(ctx).Model(syncRecord).Omit(clause.Associations).Updates(updates).Error; updateErr != nil {
+ return nil, fmt.Errorf("failed to update sync: %w", updateErr)
}
-
- // Log event
_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeGitSyncUpdate,
Severity: event.EventSeveritySuccess,
@@ -891,56 +682,42 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
})
}
- // Reconcile the dynamic job to match the new state.
- switch {
- case newAutoSync:
- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)
- if !oldAutoSync {
- // Freshly enabled — kick a run now so it doesn't wait a full interval.
- s.kickSyncInternal(ctx, syncRecord.ID)
- }
- default:
- s.unregisterSyncJobInternal(ctx, syncRecord.ID)
+ if !newAutoSync {
+ s.jobs.Unregister(ctx, syncRecord.ID)
+ return s.GetSyncByID(ctx, environmentID, id)
+ }
+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)
+ if !oldAutoSync {
+ // A freshly enabled sync runs now instead of waiting a full interval.
+ s.kickSync(ctx, syncRecord.ID, "startup")
}
return s.GetSyncByID(ctx, environmentID, id)
}
func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id string, actor user.Actor) error {
- // Stop the recurring job first, unconditionally. Even a sync whose row can no
- // longer be loaded (corrupt or environment-mismatched) must stop firing; any
- // in-flight run re-reads the row and self-cancels once it is gone.
- s.unregisterSyncJobInternal(ctx, id)
+ // Even a row that can no longer be loaded must stop firing; an in-flight run re-reads it and skips.
+ s.jobs.Unregister(ctx, id)
s.backups.cancel(id)
- // Best-effort load for the audit-event metadata. A corrupt or env-mismatched row
- // must still be deletable, so a load failure falls through to the direct delete
- // below instead of aborting.
- syncRecord, loadErr := s.getSyncRecordByIDInternal(ctx, environmentID, id)
+ // The row is loaded only for the audit event, so a corrupt or environment-mismatched row is still deletable.
+ syncRecord, loadErr := s.getSyncByID(ctx, environmentID, id, false)
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- // Clear gitops_managed_by for any project still pointing at this sync, keyed
- // on the sync id so orphaned managed flags are cleared even when the sync row
- // (and its ProjectID) could not be loaded.
+ // Keyed on the sync ID, so orphaned managed flags clear even when the row could not be loaded.
if err := tx.Model(&projectpkg.Project{}).
Where("gitops_managed_by = ?", id).
Update("gitops_managed_by", nil).Error; err != nil {
return fmt.Errorf("failed to clear gitops_managed_by: %w", err)
}
- if err :=
-
- // Delete by id only (no environment scoping). The handler already enforced the
- // delete permission; env scoping is precisely what made env-mismatched corrupt
- // rows undeletable. A zero-row delete is treated as success (idempotent).
-
- tx.Where("id = ?", id).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {
+ // The handler enforced the delete permission, so the delete is not environment-scoped, and deleting nothing succeeds.
+ if err := tx.Where("id = ?", id).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {
return fmt.Errorf("failed to delete sync: %w", err)
}
return nil
}); err != nil {
- // Re-register the recurring job only when we actually loaded an auto-sync row.
if loadErr == nil && syncRecord.AutoSync {
- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
}
return err
}
@@ -950,7 +727,6 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
return nil
}
- // Log event
_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeGitSyncDelete,
Severity: event.EventSeverityInfo,
@@ -968,15 +744,13 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
}
func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor user.Actor) (*gitops.SyncResult, error) {
- return s.performSyncAdmittedInternal(ctx, environmentID, id, actor, false)
+ return s.performSyncAdmitted(ctx, environmentID, id, actor, false)
}
-// performSyncAdmittedInternal runs one sync under the per-sync admission lease
-// and dispatches by mode. backupAdopt makes a backup run replace whatever the
-// remote holds in its backup directory.
-func (s *GitOpsSyncService) performSyncAdmittedInternal(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {
- // Coalesce overlapping runs for the same sync (scheduled fire, startup/enable
- // kick, manual trigger, webhook) so they don't race the clone/redeploy.
+// performSyncAdmitted runs one sync under its admission lease and dispatches by mode. backupAdopt makes a
+// backup replace whatever the remote holds in its backup directory.
+func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {
+ // Overlapping runs of one sync (schedule, kick, manual, webhook) must not race the clone and redeploy.
lease, admitted, err := s.jobs.TryAcquire(ctx, id)
if err != nil {
return nil, fmt.Errorf("failed to admit GitOps sync: %w", err)
@@ -1035,12 +809,8 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, id
return status, nil
}
-// CleanupLeakedScratchDirsOnStartup removes orphaned GitOps scratch directories
-// (.gitops-sync-stage-*, .gitops-backup-*, and the legacy "<name>.gitops-backup-<digits>"
-// form) left in the projects directory by a crash or container restart that interrupted
-// a sync mid-flight. These are Arcane-internal working dirs; left in place the filesystem
-// discovery imports them as phantom projects. It is safe to run at startup because it
-// executes before any sync job is registered, so nothing is mid-stage.
+// CleanupLeakedScratchDirsOnStartup removes GitOps scratch directories an interrupted sync left in the projects
+// directory, which discovery would import as phantom projects. It runs before any sync job is registered.
func (s *GitOpsSyncService) CleanupLeakedScratchDirsOnStartup(ctx context.Context) error {
projectsDir, err := s.projectService.GetProjectsDirectory(ctx)
if err != nil {
@@ -1234,38 +1004,7 @@ func (s *GitOpsSyncService) ImportSyncs(ctx context.Context, environmentID strin
return response, nil
}
-func (s *GitOpsSyncService) logSyncError(ctx context.Context, syncRecord *projectpkg.GitOpsSync, actor user.Actor, errorMsg string) {
- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
- Type: event.EventTypeGitSyncError,
- Severity: event.EventSeverityError,
- Title: "Git sync failed",
- Description: fmt.Sprintf("Failed to sync '%s': %s", syncRecord.Name, errorMsg),
- ResourceType: new("git_sync"),
- ResourceID: new(syncRecord.ID),
- ResourceName: new(syncRecord.Name),
- UserID: new(actor.ID),
- Username: new(actor.Username),
- EnvironmentID: new(syncRecord.EnvironmentID),
- })
-}
-
-// backupRuntimeInternal holds the in-memory coordination for backup syncs
-type backupRuntimeInternal struct {
- mu sync.Mutex
- branches map[string]*sync.Mutex
- timers map[string]*time.Timer
- debounce time.Duration
-}
-
-func newBackupRuntimeInternal() *backupRuntimeInternal {
- return &backupRuntimeInternal{
- branches: make(map[string]*sync.Mutex),
- timers: make(map[string]*time.Timer),
- debounce: backup.DefaultBackupSaveDebounce,
- }
-}
-
-func (r *backupRuntimeInternal) branchLock(repositoryID, branch string) *sync.Mutex {
+func (r *backupRuntime) branchLock(repositoryID, branch string) *sync.Mutex {
r.mu.Lock()
defer r.mu.Unlock()
key := repositoryID + "\x00" + branch
@@ -1277,7 +1016,7 @@ func (r *backupRuntimeInternal) branchLock(repositoryID, branch string) *sync.Mu
return lock
}
-func (r *backupRuntimeInternal) schedule(syncID string, run func()) {
+func (r *backupRuntime) schedule(syncID string, run func()) {
r.mu.Lock()
defer r.mu.Unlock()
if timer, ok := r.timers[syncID]; ok {
@@ -1291,7 +1030,7 @@ func (r *backupRuntimeInternal) schedule(syncID string, run func()) {
})
}
-func (r *backupRuntimeInternal) cancel(syncID string) {
+func (r *backupRuntime) cancel(syncID string) {
r.mu.Lock()
defer r.mu.Unlock()
if timer, ok := r.timers[syncID]; ok {
@@ -1305,11 +1044,11 @@ func (s *GitOpsSyncService) ResolveBackupConflict(ctx context.Context, environme
if req.Strategy != gitops.BackupConflictUseArcane {
return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "strategy", Err: fmt.Errorf("unsupported strategy %q", req.Strategy)})
}
- syncRecord, err := s.getBackupSyncInternal(ctx, environmentID, id)
+ syncRecord, err := s.getBackupSync(ctx, environmentID, id)
if err != nil {
return nil, err
}
- return s.performSyncAdmittedInternal(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor, true)
+ return s.performSyncAdmitted(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor, true)
}
// SubscribeProjectFileChanges marks backups pending on save and runs them after a debounce when auto backup is on.
@@ -1339,9 +1078,7 @@ func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {
syncID, environmentID := syncRecord.ID, syncRecord.EnvironmentID
s.backups.schedule(syncID, func() {
if s.jobs.Enabled() {
- if _, err := s.jobs.Scheduler().Submit(runCtx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "save"}); err != nil {
- slog.ErrorContext(runCtx, "git backup admission after save failed", "syncId", syncID, "error", err)
- }
+ s.kickSync(runCtx, syncID, "save")
return
}
go func() {
@@ -1359,7 +1096,7 @@ func (s *GitOpsSyncService) PreviewBackup(ctx context.Context, environmentID, id
previewCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
- syncRecord, err := s.getBackupSyncInternal(previewCtx, environmentID, id)
+ syncRecord, err := s.getBackupSync(previewCtx, environmentID, id)
if err != nil {
return nil, err
}
@@ -1371,7 +1108,7 @@ func (s *GitOpsSyncService) GetBackupHistory(ctx context.Context, environmentID,
historyCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
- syncRecord, err := s.getBackupSyncInternal(historyCtx, environmentID, id)
+ syncRecord, err := s.getBackupSync(historyCtx, environmentID, id)
if err != nil {
return nil, err
}
@@ -1383,7 +1120,7 @@ func (s *GitOpsSyncService) GetBackupRevision(ctx context.Context, environmentID
revisionCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
- syncRecord, err := s.getBackupSyncInternal(revisionCtx, environmentID, id)
+ syncRecord, err := s.getBackupSync(revisionCtx, environmentID, id)
if err != nil {
return nil, err
}
@@ -1409,7 +1146,7 @@ func (s *GitOpsSyncService) ReconcileInterruptedBackupsOnStartup(ctx context.Con
return nil
}
-func (s *GitOpsSyncService) getBackupSyncInternal(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
+func (s *GitOpsSyncService) getBackupSync(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
syncRecord, err := s.GetSyncByID(ctx, environmentID, id)
if err != nil {
return nil, err
diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.go
index 341ca4e2a..f186283e9 100644
--- a/backend/internal/gitops/service_test.go
+++ b/backend/internal/gitops/service_test.go
@@ -13,7 +13,6 @@ import (
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/gitops"
- "github.com/getarcaneapp/arcane/types/v2/lifecycle"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/go-git/go-billy/v5/osfs"
@@ -26,6 +25,7 @@ import (
"go.getarcane.app/kit/pkg"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -37,13 +37,15 @@ import (
projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func setupGitOpsProjectTestDBInternal(t *testing.T) *database.DB {
+func setupGitOpsProjectTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -54,7 +56,7 @@ func setupGitOpsProjectTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func newGitOpsSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newGitOpsSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
service, err := settings.NewSettingsService(ctx, db)
if err == nil {
@@ -63,7 +65,7 @@ func newGitOpsSettingsServiceForTestInternal(t testing.TB, ctx context.Context,
return service, err
}
-func newGitOpsAdmissionGateForTestInternal(t testing.TB) *runs.Admission {
+func newGitOpsAdmissionGateForTest(t testing.TB) *runs.Admission {
t.Helper()
runtime := francistest.New(t)
gate := runs.NewAdmission(runtime.Service(), t.Name())
@@ -76,10 +78,10 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *dat
t.Helper()
ctx := t.Context()
- db := setupGitOpsProjectTestDBInternal(t)
+ db := setupGitOpsProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}))
- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
projectsDir := t.TempDir()
@@ -91,15 +93,15 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *dat
return NewGitOpsSyncService(db, nil, projectService, nil, eventService, settingsService), db, projectsDir
}
-func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *testing.T) {
- gate := newGitOpsAdmissionGateForTestInternal(t)
- key := schedulertypes.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}
+func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkip(t *testing.T) {
+ gate := newGitOpsAdmissionGateForTest(t)
+ key := schedulertypes.AdmissionKey{Scope: gitOpsSyncAdmissionScope, ID: "sync-id"}
lease, admitted, err := gate.TryAcquire(t.Context(), key)
require.NoError(t, err)
require.True(t, admitted)
- service := &GitOpsSyncService{jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScopeInternal)}
- require.NoError(t, service.SetScheduler(t.Context(), &gitOpsSyncTestSchedulerInternal{}, gate))
+ service := &GitOpsSyncService{jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScope)}
+ require.NoError(t, service.SetScheduler(t.Context(), &gitOpsSyncTestScheduler{}, gate))
result, err := service.PerformSync(t.Context(), "0", "sync-id", user.Actor{})
require.NoError(t, err)
require.False(t, result.Success)
@@ -107,36 +109,36 @@ func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *
lease.Release(t.Context())
}
-type gitOpsSyncTestSchedulerInternal struct {
+type gitOpsSyncTestScheduler struct {
submitted []schedulertypes.Request
added []string
removed []string
}
-func (s *gitOpsSyncTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
+func (s *gitOpsSyncTestScheduler) AddJob(_ context.Context, job schedulertypes.Job) error {
s.added = append(s.added, job.Name())
return nil
}
-func (s *gitOpsSyncTestSchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *gitOpsSyncTestScheduler) RemoveJob(_ context.Context, name string) {
s.removed = append(s.removed, name)
}
-func (s *gitOpsSyncTestSchedulerInternal) HasJob(_ string) bool {
+func (s *gitOpsSyncTestScheduler) HasJob(_ string) bool {
return false
}
-// setupGitOpsSyncRemoteTestServiceInternal wires a sync service with a real
+// setupGitOpsSyncRemoteTestService wires a sync service with a real
// scheduler registry over a bare remote "repo-1" whose main branch holds files.
-func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]string) (*GitOpsSyncService, *database.DB, string, *gitOpsSyncTestSchedulerInternal) {
+func setupGitOpsSyncRemoteTestService(t *testing.T, files map[string]string) (*GitOpsSyncService, *database.DB, string, *gitOpsSyncTestScheduler) {
t.Helper()
- installBackupTestTransportInternal()
+ installBackupTestTransport()
ctx := t.Context()
- db := setupGitOpsProjectTestDBInternal(t)
+ db := setupGitOpsProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &gitrepo.GitRepository{}))
- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
projectsDir := t.TempDir()
@@ -147,15 +149,15 @@ func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]str
repoService := gitrepo.NewGitRepositoryService(db, t.TempDir(), eventService, settingsService)
service := NewGitOpsSyncService(db, repoService, projectService, nil, eventService, settingsService)
- testScheduler := &gitOpsSyncTestSchedulerInternal{}
- require.NoError(t, service.SetScheduler(ctx, testScheduler, newGitOpsAdmissionGateForTestInternal(t)))
+ testScheduler := &gitOpsSyncTestScheduler{}
+ require.NoError(t, service.SetScheduler(ctx, testScheduler, newGitOpsAdmissionGateForTest(t)))
bare := filepath.Join(t.TempDir(), "sync.git")
_, err = gogit.PlainInit(bare, true)
require.NoError(t, err)
repoURL := "http://localhost" + bare
require.NoError(t, db.Create(&gitrepo.GitRepository{ID: "repo-1", Name: "sync-remote", URL: repoURL, AuthType: "none", Enabled: true}).Error)
- pushRemoteCommitInternal(t, git.NewClient(t.TempDir()), repoURL, "seed", files)
+ pushRemoteCommit(t, git.NewClient(t.TempDir()), repoURL, "seed", files)
return service, db, projectsDir, testScheduler
}
@@ -165,7 +167,7 @@ func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]str
// broken binding, not a "-N" duplicate.
func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir, _ := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
+ svc, db, projectsDir, _ := setupGitOpsSyncRemoteTestService(t, map[string]string{"docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
require.NoError(t, os.MkdirAll(filepath.Join(projectsDir, "Dozzle"), 0o755))
syncRecord := &projectpkg.GitOpsSync{
@@ -194,7 +196,7 @@ func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(t
func TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
missingProjectID := "missing-project"
syncRecord := &projectpkg.GitOpsSync{
@@ -237,7 +239,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *
func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProjectRecoveryAmbiguous(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/media/radarr.yaml": "services:\n app:\n image: lscr.io/linuxserver/radarr:latest\n"})
+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/media/radarr.yaml": "services:\n app:\n image: lscr.io/linuxserver/radarr:latest\n"})
for _, dirName := range []string{"Radarr-3", "Radarr-30"} {
projectPath := filepath.Join(projectsDir, dirName)
@@ -281,9 +283,9 @@ func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProject
assert.Contains(t, testScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncRecord.ID)
}
-func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissing(t *testing.T) {
+func TestGitOpsSyncService_GetOrCreateProject_FailsWhenBoundProjectMissing(t *testing.T) {
ctx := t.Context()
- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})
missingProjectID := "missing-project"
syncRecord := &projectpkg.GitOpsSync{
@@ -324,23 +326,6 @@ func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissi
assert.Contains(t, testScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncRecord.ID)
}
-func TestApplyLifecycleFieldsToSyncInternal_DefaultsPreDeployTimeout(t *testing.T) {
- var syncRecord projectpkg.GitOpsSync
-
- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleConfigInputInternal{})
-
- require.Equal(t, lifecycle.DefaultTimeoutSec, syncRecord.PreDeployTimeoutSec)
-}
-
-func TestApplyLifecycleFieldsToSyncInternal_UsesExplicitPreDeployTimeout(t *testing.T) {
- timeoutSec := 90
- var syncRecord projectpkg.GitOpsSync
-
- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleConfigInputInternal{timeoutSec: &timeoutSec})
-
- require.Equal(t, timeoutSec, syncRecord.PreDeployTimeoutSec)
-}
-
func TestGitOpsSyncService_GetSyncByID_ReturnsNotFoundError(t *testing.T) {
ctx := t.Context()
svc, _, _ := setupGitOpsSyncDirectoryTestService(t)
@@ -434,8 +419,8 @@ func TestGitOpsSyncService_RegisterAutoSyncJobsOnStartup_SkipsOrphans(t *testing
LastSyncAt: &now,
}).Error)
- scheduler := &gitOpsSyncTestSchedulerInternal{}
- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))
+ scheduler := &gitOpsSyncTestScheduler{}
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))
svc.RegisterAutoSyncJobsOnStartup(ctx)
require.Equal(t, []string{entityjobs.GitOpsSyncJobPrefix + "sync-live"}, scheduler.added)
@@ -472,8 +457,8 @@ func TestGitOpsSyncService_DeleteSync_DeletesStaleProjectReference(t *testing.T)
func TestGitOpsSyncService_DeleteSync_SucceedsWhenEnvironmentMismatched(t *testing.T) {
ctx := t.Context()
svc, db, _ := setupGitOpsSyncDirectoryTestService(t)
- scheduler := &gitOpsSyncTestSchedulerInternal{}
- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))
+ scheduler := &gitOpsSyncTestScheduler{}
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))
syncRecord := &projectpkg.GitOpsSync{
ID: "sync-env-mismatch",
@@ -523,18 +508,22 @@ func TestGitOpsSyncService_DeleteSync_ClearsOrphanedManagedFlag(t *testing.T) {
assert.Nil(t, got.GitOpsManagedBy)
}
-// TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync verifies a scheduled
-// run whose row no longer exists (e.g. deleted out-of-band via raw SQL) unregisters
-// its own job instead of firing forever.
-func TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync(t *testing.T) {
+// TestGitOpsSyncWorkflow_UnregistersMissingSync verifies a scheduled run whose row no longer exists
+// (e.g. deleted out-of-band via raw SQL) unregisters its own job instead of firing forever.
+func TestGitOpsSyncWorkflow_UnregistersMissingSync(t *testing.T) {
ctx := t.Context()
- svc, _, _ := setupGitOpsSyncDirectoryTestService(t)
- scheduler := &gitOpsSyncTestSchedulerInternal{}
- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))
-
- _, scheduledSyncErr := svc.runScheduledSyncInternal(ctx, "0", "ghost-sync")
- require.NoError(t, scheduledSyncErr)
-
+ svc, db, _ := setupGitOpsSyncDirectoryTestService(t)
+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
+ scheduler := &gitOpsSyncTestScheduler{}
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))
+
+ outcome, err := harness.Engine.Run(ctx, svc.syncWorkflow, syncTarget{EnvironmentID: "0", SyncID: "ghost-sync"}, activitylib.StartRequest{}, nil)
+ require.NoError(t, err)
+ assert.Equal(t, schedulertypes.Skipped, outcome.Status)
assert.Contains(t, scheduler.removed, entityjobs.GitOpsSyncJobPrefix+"ghost-sync")
}
@@ -664,8 +653,8 @@ func TestGitOpsSyncService_ReconcileDirectorySyncProjectsOnStartup_SkipsAmbiguou
func TestGitOpsSyncService_GetEnvironmentSyncLimits(t *testing.T) {
ctx := t.Context()
- db := setupGitOpsProjectTestDBInternal(t)
- settingsSvc, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ db := setupGitOpsProjectTestDB(t)
+ settingsSvc, err := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 123))
@@ -687,8 +676,8 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "")
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "")
- db := setupGitOpsProjectTestDBInternal(t)
- settingsSvc, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ db := setupGitOpsProjectTestDB(t)
+ settingsSvc, err := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 200))
@@ -743,7 +732,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_FILES", "10000")
t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "1024")
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "12")
- settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, svcErr)
svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}
@@ -764,7 +753,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_FILES", "0")
t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "0")
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "0")
- settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, svcErr)
svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}
@@ -795,14 +784,14 @@ func setupLifecycleValidationService(t *testing.T) (*GitOpsSyncService, context.
func TestValidateLifecycleConfig_AllNilNoError(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{}))
+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{}))
}
func TestValidateLifecycleConfig_RejectsWhenGloballyDisabled(t *testing.T) {
svc, _, _ := setupGitOpsSyncDirectoryTestService(t)
ctx := t.Context()
// lifecycleEnabled defaults to false; do not enable.
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new("scripts/deploy.sh"),
runnerImage: new("alpine:latest"),
})
@@ -812,7 +801,7 @@ func TestValidateLifecycleConfig_RejectsWhenGloballyDisabled(t *testing.T) {
func TestValidateLifecycleConfig_RejectsAbsoluteScriptPath(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new("/etc/passwd"),
runnerImage: new("alpine:latest"),
})
@@ -822,7 +811,7 @@ func TestValidateLifecycleConfig_RejectsAbsoluteScriptPath(t *testing.T) {
func TestValidateLifecycleConfig_RejectsTraversalScriptPath(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new("../outside.sh"),
runnerImage: new("alpine:latest"),
})
@@ -836,7 +825,7 @@ func TestValidateLifecycleConfig_RejectsOverlongScriptPath(t *testing.T) {
for i := range long {
long[i] = 'a'
}
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new(string(long)),
runnerImage: new("alpine:latest"),
})
@@ -847,7 +836,7 @@ func TestValidateLifecycleConfig_RejectsOverlongScriptPath(t *testing.T) {
func TestValidateLifecycleConfig_RejectsScriptWithoutRunnerImageOnCreate(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleDefaultRunnerImage", " "))
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new("scripts/deploy.sh"),
})
require.Error(t, err)
@@ -857,7 +846,7 @@ func TestValidateLifecycleConfig_RejectsScriptWithoutRunnerImageOnCreate(t *test
func TestValidateLifecycleConfig_AcceptsScriptWithDefaultRunnerImageOnCreate(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleDefaultRunnerImage", "alpine:latest"))
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
targetType: new("project"),
scriptPath: new("scripts/deploy.sh"),
syncDirectory: new(true),
@@ -868,7 +857,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithExistingRunnerImageOnUpdate(t
svc, ctx := setupLifecycleValidationService(t)
existing := &projectpkg.GitOpsSync{SyncDirectory: true}
existing.PreDeployRunnerImage = new("alpine:latest")
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{
scriptPath: new("scripts/deploy.sh"),
}))
}
@@ -876,7 +865,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithExistingRunnerImageOnUpdate(t
func TestValidateLifecycleConfig_RejectsTimeoutZeroOrNegative(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
for _, v := range []int{0, -1, -3600} {
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
timeoutSec: new(v),
})
require.Errorf(t, err, "expected error for timeoutSec=%d", v)
@@ -887,7 +876,7 @@ func TestValidateLifecycleConfig_RejectsTimeoutZeroOrNegative(t *testing.T) {
func TestValidateLifecycleConfig_RejectsTimeoutAboveSettingCap(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleMaxTimeoutSec", "120"))
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
timeoutSec: new(300),
})
require.Error(t, err)
@@ -896,7 +885,7 @@ func TestValidateLifecycleConfig_RejectsTimeoutAboveSettingCap(t *testing.T) {
func TestValidateLifecycleConfig_RejectsInvalidEnvKey(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
env: new("FOO-BAR=baz"),
})
require.Error(t, err)
@@ -904,14 +893,14 @@ func TestValidateLifecycleConfig_RejectsInvalidEnvKey(t *testing.T) {
func TestValidateLifecycleConfig_AcceptsValidEnv(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
env: new("FOO=bar\nBAZ_2=qux"),
}))
}
func TestValidateLifecycleConfig_RejectsRelativeMountSource(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
extraMounts: new("relative/path:/in/container"),
})
require.Error(t, err)
@@ -920,7 +909,7 @@ func TestValidateLifecycleConfig_RejectsRelativeMountSource(t *testing.T) {
func TestValidateLifecycleConfig_RejectsRelativeMountTarget(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
extraMounts: new("/host/path:relative/target"),
})
require.Error(t, err)
@@ -933,14 +922,14 @@ func TestValidateLifecycleConfig_AllowsClearingScriptWithoutImage(t *testing.T)
existing.PreDeployScriptPath = new("scripts/old.sh")
existing.PreDeployRunnerImage = new("alpine:latest")
// User clears the script (empty string in update); image clear is implied not required.
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{
scriptPath: new(""),
}))
}
func TestValidateLifecycleConfig_RejectsScriptWithoutSyncDirectoryOnCreate(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
scriptPath: new("scripts/deploy.sh"),
runnerImage: new("alpine:latest"),
syncDirectory: new(false),
@@ -954,7 +943,7 @@ func TestValidateLifecycleConfig_RejectsScriptWithoutSyncDirectoryOnCreate(t *te
func TestValidateLifecycleConfig_AcceptsScriptWithSyncDirectoryOnCreate(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
targetType: new("project"),
scriptPath: new("scripts/deploy.sh"),
runnerImage: new("alpine:latest"),
@@ -964,7 +953,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithSyncDirectoryOnCreate(t *testi
func TestValidateLifecycleConfig_RejectsLifecycleHookForSwarmStack(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{
targetType: new("swarm_stack"),
scriptPath: new("scripts/deploy.sh"),
runnerImage: new("alpine:latest"),
@@ -983,7 +972,7 @@ func TestValidateLifecycleConfig_RejectsSwarmTargetChangeWithExistingLifecycleHo
existing := &projectpkg.GitOpsSync{TargetType: "project", SyncDirectory: true}
existing.PreDeployScriptPath = new("scripts/deploy.sh")
existing.PreDeployRunnerImage = new("alpine:latest")
- err := svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{
targetType: new("swarm_stack"),
})
require.Error(t, err)
@@ -997,7 +986,7 @@ func TestValidateLifecycleConfig_RejectsSwarmTargetChangeWithExistingLifecycleHo
func TestValidateLifecycleConfig_AcceptsScriptWhenExistingSyncHasSyncDirectory(t *testing.T) {
svc, ctx := setupLifecycleValidationService(t)
existing := &projectpkg.GitOpsSync{SyncDirectory: true}
- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{
+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{
scriptPath: new("scripts/deploy.sh"),
runnerImage: new("alpine:latest"),
}))
@@ -1009,7 +998,7 @@ func TestValidateLifecycleConfig_RejectsSyncDirectoryToggleOffWhileScriptStillSe
existing.PreDeployScriptPath = new("scripts/deploy.sh")
existing.PreDeployRunnerImage = new("alpine:latest")
// Admin toggles syncDirectory off without clearing the script — should be rejected.
- err := svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{
+ err := svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{
syncDirectory: new(false),
})
require.Error(t, err)
@@ -1029,27 +1018,27 @@ func TestRedeployAfterSyncFailedError_FormatAndUnwrap(t *testing.T) {
require.ErrorIs(t, err, common.ErrRedeployAfterSyncFailed)
}
-func (s *gitOpsSyncTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+func (s *gitOpsSyncTestScheduler) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
s.submitted = append(s.submitted, request)
return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
}
-var installBackupTestTransportOnceInternal sync.Once
+var installBackupTestTransportOnce sync.Once
-// installBackupTestTransportInternal serves bare repositories on disk over the
+// installBackupTestTransport serves bare repositories on disk over the
// "http" scheme so backups push without a network or the git binary.
-func installBackupTestTransportInternal() {
- installBackupTestTransportOnceInternal.Do(func() {
+func installBackupTestTransport() {
+ installBackupTestTransportOnce.Do(func() {
client.InstallProtocol("http", server.NewClient(server.NewFilesystemLoader(osfs.New("/"))))
})
}
-// backupTestEnvInternal is one wired-up backup fixture: a bare remote, a git
+// backupTestEnv is one wired-up backup fixture: a bare remote, a git
// repository row, a project directory and the sync service under test.
-type backupTestEnvInternal struct {
+type backupTestEnv struct {
service *GitOpsSyncService
db *database.DB
- scheduler *gitOpsBackupTestSchedulerInternal
+ scheduler *gitOpsBackupTestScheduler
projectsDir string
projectPath string
project *projectpkg.Project
@@ -1057,15 +1046,15 @@ type backupTestEnvInternal struct {
remote *git.Client
}
-// gitOpsBackupTestSchedulerInternal runs a submitted job body inline so the
+// gitOpsBackupTestScheduler runs a submitted job body inline so the
// save-debounce path reaches PerformSync the way the real scheduler does.
-type gitOpsBackupTestSchedulerInternal struct {
+type gitOpsBackupTestScheduler struct {
mu sync.Mutex
jobs map[string]schedulertypes.Job
submitted []schedulertypes.Request
}
-func (s *gitOpsBackupTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
+func (s *gitOpsBackupTestScheduler) AddJob(_ context.Context, job schedulertypes.Job) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.jobs == nil {
@@ -1075,20 +1064,20 @@ func (s *gitOpsBackupTestSchedulerInternal) AddJob(_ context.Context, job schedu
return nil
}
-func (s *gitOpsBackupTestSchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *gitOpsBackupTestScheduler) RemoveJob(_ context.Context, name string) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.jobs, name)
}
-func (s *gitOpsBackupTestSchedulerInternal) HasJob(name string) bool {
+func (s *gitOpsBackupTestScheduler) HasJob(name string) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.jobs[name]
return ok
}
-func (s *gitOpsBackupTestSchedulerInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+func (s *gitOpsBackupTestScheduler) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
s.mu.Lock()
s.submitted = append(s.submitted, request)
job := s.jobs[request.JobID]
@@ -1099,21 +1088,21 @@ func (s *gitOpsBackupTestSchedulerInternal) Submit(ctx context.Context, request
return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
}
-func (s *gitOpsBackupTestSchedulerInternal) submitCount() int {
+func (s *gitOpsBackupTestScheduler) submitCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.submitted)
}
-func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
+func setupGitOpsBackupTestService(t *testing.T) *backupTestEnv {
t.Helper()
- installBackupTestTransportInternal()
+ installBackupTestTransport()
ctx := t.Context()
- db := setupGitOpsProjectTestDBInternal(t)
- require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &projectpkg.ProjectTag{}, &gitrepo.GitRepository{}, &environment.Environment{}))
+ db := setupGitOpsProjectTestDB(t)
+ require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &projectpkg.ProjectTag{}, &gitrepo.GitRepository{}, &environment.Environment{}, &activity.Activity{}, &activity.ActivityMessage{}))
- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)
+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)
require.NoError(t, err)
projectsDir := t.TempDir()
@@ -1124,8 +1113,12 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
repoService := gitrepo.NewGitRepositoryService(db, t.TempDir(), eventService, settingsService)
service := NewGitOpsSyncService(db, repoService, projectService, nil, eventService, settingsService)
- scheduler := &gitOpsBackupTestSchedulerInternal{}
- require.NoError(t, service.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))
+ // The test scheduler runs submitted jobs inline, which runs the sync workflow.
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ require.NoError(t, service.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ scheduler := &gitOpsBackupTestScheduler{}
+ require.NoError(t, service.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))
bare := filepath.Join(t.TempDir(), "backups.git")
_, err = gogit.PlainInit(bare, true)
@@ -1142,7 +1135,7 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
projectPath := filepath.Join(projectsDir, "demo-project")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
- writeBackupProjectFileInternal(t, projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.27-alpine\n")
+ writeBackupProjectFile(t, projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.27-alpine\n")
project := &projectpkg.Project{
ID: "proj-backup",
@@ -1153,7 +1146,7 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
}
require.NoError(t, db.Create(project).Error)
- return &backupTestEnvInternal{
+ return &backupTestEnv{
service: service,
db: db,
scheduler: scheduler,
@@ -1165,14 +1158,14 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
}
}
-func writeBackupProjectFileInternal(t *testing.T, root, relative, content string) {
+func writeBackupProjectFile(t *testing.T, root, relative, content string) {
t.Helper()
target := filepath.Join(root, filepath.FromSlash(relative))
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
require.NoError(t, os.WriteFile(target, []byte(content), 0o644))
}
-func (e *backupTestEnvInternal) createBackupInternal(t *testing.T, req gitops.CreateSyncRequest) *projectpkg.GitOpsSync {
+func (e *backupTestEnv) createBackup(t *testing.T, req gitops.CreateSyncRequest) *projectpkg.GitOpsSync {
t.Helper()
req.Name = cmp.Or(req.Name, "demo-backup")
req.RepositoryID = cmp.Or(req.RepositoryID, "repo-backup")
@@ -1187,15 +1180,15 @@ func (e *backupTestEnvInternal) createBackupInternal(t *testing.T, req gitops.Cr
return syncRecord
}
-func (e *backupTestEnvInternal) reloadInternal(t *testing.T, id string) *projectpkg.GitOpsSync {
+func (e *backupTestEnv) reload(t *testing.T, id string) *projectpkg.GitOpsSync {
t.Helper()
var syncRecord projectpkg.GitOpsSync
require.NoError(t, e.db.Where("id = ?", id).First(&syncRecord).Error)
return &syncRecord
}
-// checkoutRemoteInternal clones the backup branch so the pushed tree can be inspected.
-func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {
+// checkoutRemote clones the backup branch so the pushed tree can be inspected.
+func (e *backupTestEnv) checkoutRemote(t *testing.T) string {
t.Helper()
repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"}, 0)
require.NoError(t, err)
@@ -1203,7 +1196,7 @@ func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {
return repoPath
}
-func (e *backupTestEnvInternal) remoteHeadInternal(t *testing.T) string {
+func (e *backupTestEnv) remoteHead(t *testing.T) string {
t.Helper()
head, exists, err := e.remote.RemoteBranchHead(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"})
require.NoError(t, err)
@@ -1211,8 +1204,8 @@ func (e *backupTestEnvInternal) remoteHeadInternal(t *testing.T) string {
return head
}
-// pushRemoteCommitInternal commits directly to the branch as another writer.
-func pushRemoteCommitInternal(t *testing.T, remote *git.Client, repoURL, message string, files map[string]string) {
+// pushRemoteCommit commits directly to the branch as another writer.
+func pushRemoteCommit(t *testing.T, remote *git.Client, repoURL, message string, files map[string]string) {
t.Helper()
auth := git.AuthConfig{AuthType: "none"}
checkout, err := remote.CheckoutForWrite(t.Context(), repoURL, "main", auth)
@@ -1228,16 +1221,16 @@ func pushRemoteCommitInternal(t *testing.T, remote *git.Client, repoURL, message
}
func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")
+ env := setupGitOpsBackupTestService(t)
+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})
require.Equal(t, gitops.SyncModeBackup, syncRecord.Mode)
require.Equal(t, "backups/demo", syncRecord.BackupDirectory)
require.Equal(t, "backups/demo/compose.yaml", syncRecord.ComposePath)
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
require.NotNil(t, stored.LastSyncStatus)
assert.Equal(t, "success", *stored.LastSyncStatus)
assert.NotNil(t, stored.LastBackupAt)
@@ -1245,10 +1238,10 @@ func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {
assert.False(t, stored.BackupConflict)
assert.Nil(t, stored.BackupFailureReason)
require.NotNil(t, stored.LastSyncCommit)
- assert.Equal(t, env.remoteHeadInternal(t), *stored.LastSyncCommit)
+ assert.Equal(t, env.remoteHead(t), *stored.LastSyncCommit)
require.NotNil(t, stored.LastBackupSnapshot)
- repoPath := env.checkoutRemoteInternal(t)
+ repoPath := env.checkoutRemote(t)
composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))
require.NoError(t, err)
assert.Contains(t, string(composeBytes), "nginx:1.27-alpine")
@@ -1262,32 +1255,32 @@ func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {
}
func TestGitOpsBackup_UnchangedContentMakesNoNewCommit(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
- firstHead := env.remoteHeadInternal(t)
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
+ firstHead := env.remoteHead(t)
result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)
require.NoError(t, err)
assert.True(t, result.Success)
assert.Contains(t, result.Message, "already contains")
- assert.Equal(t, firstHead, env.remoteHeadInternal(t))
+ assert.Equal(t, firstHead, env.remoteHead(t))
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
require.NotNil(t, stored.LastSyncStatus)
assert.Equal(t, "success", *stored.LastSyncStatus)
assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())
}
func TestGitOpsBackup_AddsAndRemovesFilesAndKeepsUnrelatedRemoteFiles(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")
- writeBackupProjectFileInternal(t, env.projectPath, "config/extra.conf", "extra = 1\n")
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})
+ env := setupGitOpsBackupTestService(t)
+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")
+ writeBackupProjectFile(t, env.projectPath, "config/extra.conf", "extra = 1\n")
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})
- pushRemoteCommitInternal(t, env.remote, env.repoURL, "unrelated docs", map[string]string{"docs/readme.md": "docs\n"})
+ pushRemoteCommit(t, env.remote, env.repoURL, "unrelated docs", map[string]string{"docs/readme.md": "docs\n"})
require.NoError(t, os.Remove(filepath.Join(env.projectPath, "config", "extra.conf")))
- writeBackupProjectFileInternal(t, env.projectPath, "scripts/run.sh", "#!/bin/sh\necho hi\n")
+ writeBackupProjectFile(t, env.projectPath, "scripts/run.sh", "#!/bin/sh\necho hi\n")
_, err := env.service.UpdateSync(t.Context(), "0", syncRecord.ID, gitops.UpdateSyncRequest{
BackupPaths: []string{"compose.yaml", "config", "scripts"},
@@ -1298,7 +1291,7 @@ func TestGitOpsBackup_AddsAndRemovesFilesAndKeepsUnrelatedRemoteFiles(t *testing
require.NoError(t, err)
require.True(t, result.Success)
- repoPath := env.checkoutRemoteInternal(t)
+ repoPath := env.checkoutRemote(t)
assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "scripts", "run.sh"))
assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "config", "app.conf"))
assert.FileExists(t, filepath.Join(repoPath, "docs", "readme.md"))
@@ -1316,15 +1309,15 @@ func TestGitOpsBackup_EnvFilesOnlyIncludedWhenListedExplicitly(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- writeBackupProjectFileInternal(t, env.projectPath, ".env", "TOKEN=secret\n")
- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")
- writeBackupProjectFileInternal(t, env.projectPath, "config/.env", "TOKEN=nested\n")
- writeBackupProjectFileInternal(t, env.projectPath, "config/staging.env", "TOKEN=staging\n")
+ env := setupGitOpsBackupTestService(t)
+ writeBackupProjectFile(t, env.projectPath, ".env", "TOKEN=secret\n")
+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")
+ writeBackupProjectFile(t, env.projectPath, "config/.env", "TOKEN=nested\n")
+ writeBackupProjectFile(t, env.projectPath, "config/staging.env", "TOKEN=staging\n")
- env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: test.paths})
+ env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: test.paths})
- repoPath := env.checkoutRemoteInternal(t)
+ repoPath := env.checkoutRemote(t)
assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "compose.yaml"))
assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "config", "app.conf"))
assert.NoFileExists(t, filepath.Join(repoPath, "backups", "demo", "config", ".env"))
@@ -1339,19 +1332,19 @@ func TestGitOpsBackup_EnvFilesOnlyIncludedWhenListedExplicitly(t *testing.T) {
}
func TestGitOpsBackup_RemoteEditConflictsThenResolvesWithArcane(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
- pushRemoteCommitInternal(t, env.remote, env.repoURL, "edited backup outside arcane", map[string]string{
+ pushRemoteCommit(t, env.remote, env.repoURL, "edited backup outside arcane", map[string]string{
"backups/demo/compose.yaml": "services:\n app:\n image: tampered\n",
})
- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.28-alpine\n")
+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.28-alpine\n")
_, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)
require.Error(t, err)
require.ErrorIs(t, err, common.ErrConflict)
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
assert.True(t, stored.BackupConflict)
assert.Equal(t, gitops.BackupStateNeedsAttention, stored.BackupState())
require.NotNil(t, stored.BackupFailureReason)
@@ -1368,26 +1361,26 @@ func TestGitOpsBackup_RemoteEditConflictsThenResolvesWithArcane(t *testing.T) {
require.NoError(t, err)
require.True(t, result.Success)
- resolved := env.reloadInternal(t, syncRecord.ID)
+ resolved := env.reload(t, syncRecord.ID)
assert.False(t, resolved.BackupConflict)
assert.Nil(t, resolved.BackupFailureReason)
assert.Equal(t, gitops.BackupStateBackedUp, resolved.BackupState())
- repoPath := env.checkoutRemoteInternal(t)
+ repoPath := env.checkoutRemote(t)
composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))
require.NoError(t, err)
assert.Contains(t, string(composeBytes), "nginx:1.28-alpine")
}
func TestGitOpsBackup_OccupiedDestinationNeedsAttention(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- pushRemoteCommitInternal(t, env.remote, env.repoURL, "pre-existing files", map[string]string{
+ env := setupGitOpsBackupTestService(t)
+ pushRemoteCommit(t, env.remote, env.repoURL, "pre-existing files", map[string]string{
"backups/demo/unrelated.txt": "not an arcane backup\n",
})
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
assert.True(t, stored.BackupConflict)
assert.Equal(t, gitops.BackupStateNeedsAttention, stored.BackupState())
require.NotNil(t, stored.BackupFailureReason)
@@ -1396,9 +1389,9 @@ func TestGitOpsBackup_OccupiedDestinationNeedsAttention(t *testing.T) {
}
func TestGitOpsBackup_AdoptsMatchingRemoteWithoutSnapshot(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
- head := env.remoteHeadInternal(t)
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
+ head := env.remoteHead(t)
require.NoError(t, env.db.Model(&projectpkg.GitOpsSync{}).Where("id = ?", syncRecord.ID).
Update("last_backup_snapshot", nil).Error)
@@ -1406,25 +1399,25 @@ func TestGitOpsBackup_AdoptsMatchingRemoteWithoutSnapshot(t *testing.T) {
result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)
require.NoError(t, err)
assert.True(t, result.Success)
- assert.Equal(t, head, env.remoteHeadInternal(t))
+ assert.Equal(t, head, env.remoteHead(t))
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
assert.False(t, stored.BackupConflict)
assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())
}
func TestGitOpsBackup_SucceedsOnTopOfAnotherWritersCommit(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
- pushRemoteCommitInternal(t, env.remote, env.repoURL, "other writer", map[string]string{"other/notes.txt": "notes\n"})
- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
+ pushRemoteCommit(t, env.remote, env.repoURL, "other writer", map[string]string{"other/notes.txt": "notes\n"})
+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)
require.NoError(t, err)
require.True(t, result.Success)
- repoPath := env.checkoutRemoteInternal(t)
+ repoPath := env.checkoutRemote(t)
assert.FileExists(t, filepath.Join(repoPath, "other", "notes.txt"))
composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))
require.NoError(t, err)
@@ -1437,13 +1430,13 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {
tests := []struct {
name string
- mutate func(t *testing.T, env *backupTestEnvInternal)
+ mutate func(t *testing.T, env *backupTestEnv)
request gitops.CreateSyncRequest
wantErr error
}{
{
name: "project deployed from git",
- mutate: func(t *testing.T, env *backupTestEnvInternal) {
+ mutate: func(t *testing.T, env *backupTestEnv) {
t.Helper()
require.NoError(t, env.db.Model(&projectpkg.Project{}).Where("id = ?", env.project.ID).
Update("gitops_managed_by", "sync-deploy").Error)
@@ -1452,16 +1445,16 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {
},
{
name: "second backup for the same project",
- mutate: func(t *testing.T, env *backupTestEnvInternal) {
+ mutate: func(t *testing.T, env *backupTestEnv) {
t.Helper()
- env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env.createBackup(t, gitops.CreateSyncRequest{})
},
request: gitops.CreateSyncRequest{Name: "second", BackupDirectory: "backups/other"},
wantErr: common.ErrConflict,
},
{
name: "overlapping destination on the same branch",
- mutate: func(t *testing.T, env *backupTestEnvInternal) {
+ mutate: func(t *testing.T, env *backupTestEnv) {
t.Helper()
other := &projectpkg.Project{
ID: "proj-other",
@@ -1470,9 +1463,9 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {
Path: filepath.Join(env.projectsDir, "other-project"),
Status: projectpkg.ProjectStatusStopped,
}
- writeBackupProjectFileInternal(t, other.Path, "compose.yaml", "services: {}\n")
+ writeBackupProjectFile(t, other.Path, "compose.yaml", "services: {}\n")
require.NoError(t, env.db.Create(other).Error)
- env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env.createBackup(t, gitops.CreateSyncRequest{})
env.project = other
},
request: gitops.CreateSyncRequest{Name: "nested", BackupDirectory: "backups/demo/nested"},
@@ -1492,8 +1485,8 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")
+ env := setupGitOpsBackupTestService(t)
+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")
if test.mutate != nil {
test.mutate(t, env)
}
@@ -1513,7 +1506,7 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {
}
func TestGitOpsBackup_CreateWithoutProjectIsRejected(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
+ env := setupGitOpsBackupTestService(t)
_, err := env.service.CreateSync(t.Context(), "0", gitops.CreateSyncRequest{
Name: "demo-backup",
@@ -1528,14 +1521,14 @@ func TestGitOpsBackup_CreateWithoutProjectIsRejected(t *testing.T) {
}
func TestGitOpsBackup_SaveSignalMarksPendingAndRunsAfterDebounce(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
- firstHead := env.remoteHeadInternal(t)
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
+ firstHead := env.remoteHead(t)
env.service.backups.debounce = 50 * time.Millisecond
env.service.SubscribeProjectFileChanges(t.Context())
- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
env.service.projectService.FilesChanged.Publish(env.project.ID)
require.Eventually(t, func() bool {
@@ -1543,23 +1536,23 @@ func TestGitOpsBackup_SaveSignalMarksPendingAndRunsAfterDebounce(t *testing.T) {
return err == nil && exists && head != firstHead
}, 5*time.Second, 25*time.Millisecond)
require.Positive(t, env.scheduler.submitCount())
- require.NotEqual(t, firstHead, env.remoteHeadInternal(t))
+ require.NotEqual(t, firstHead, env.remoteHead(t))
require.Eventually(t, func() bool {
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
return stored.LastSyncStatus != nil && *stored.LastSyncStatus == "success" && !stored.BackupPending
}, 5*time.Second, 25*time.Millisecond)
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
require.NotNil(t, stored.LastBackupAt)
assert.False(t, stored.BackupPending)
assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())
}
func TestGitOpsBackup_SaveSignalOnlyMarksPendingWhenAutoSyncIsOff(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
- head := env.remoteHeadInternal(t)
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
+ head := env.remoteHead(t)
autoSync := false
_, err := env.service.UpdateSync(t.Context(), "0", syncRecord.ID, gitops.UpdateSyncRequest{AutoSync: &autoSync}, user.SystemUser)
@@ -1568,31 +1561,31 @@ func TestGitOpsBackup_SaveSignalOnlyMarksPendingWhenAutoSyncIsOff(t *testing.T)
env.service.backups.debounce = 50 * time.Millisecond
env.service.SubscribeProjectFileChanges(t.Context())
- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")
env.service.projectService.FilesChanged.Publish(env.project.ID)
require.Eventually(t, func() bool {
- return env.reloadInternal(t, syncRecord.ID).BackupPending
+ return env.reload(t, syncRecord.ID).BackupPending
}, 5*time.Second, 25*time.Millisecond)
time.Sleep(300 * time.Millisecond)
- assert.Equal(t, head, env.remoteHeadInternal(t))
+ assert.Equal(t, head, env.remoteHead(t))
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
assert.True(t, stored.BackupPending)
assert.Equal(t, gitops.BackupStatePaused, stored.BackupState())
}
func TestGitOpsBackup_ReconcileInterruptedBackupsOnStartup(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
require.NoError(t, env.db.Model(&projectpkg.GitOpsSync{}).Where("id = ?", syncRecord.ID).
Update("last_sync_status", backup.BackupStatusRunning).Error)
require.NoError(t, env.service.ReconcileInterruptedBackupsOnStartup(t.Context()))
- stored := env.reloadInternal(t, syncRecord.ID)
+ stored := env.reload(t, syncRecord.ID)
require.NotNil(t, stored.LastSyncStatus)
assert.Equal(t, "failed", *stored.LastSyncStatus)
assert.True(t, stored.BackupPending)
@@ -1601,8 +1594,8 @@ func TestGitOpsBackup_ReconcileInterruptedBackupsOnStartup(t *testing.T) {
}
func TestGitOpsBackup_DeletingProjectRemovesBackupSync(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})
+ env := setupGitOpsBackupTestService(t)
+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})
require.NoError(t, env.service.projectService.DestroyProject(t.Context(), env.project.ID, true, false, user.SystemUser))
@@ -1612,11 +1605,11 @@ func TestGitOpsBackup_DeletingProjectRemovesBackupSync(t *testing.T) {
}
func TestGitOpsDeploy_LinksExistingProject(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
+ env := setupGitOpsBackupTestService(t)
ctx := t.Context()
remoteCompose := "services:\n app:\n image: nginx:1.28-alpine\n"
- pushRemoteCommitInternal(t, env.remote, env.repoURL, "seed", map[string]string{"apps/demo/compose.yaml": remoteCompose})
- writeBackupProjectFileInternal(t, env.projectPath, ".env", "TOKEN=keep-me\n")
+ pushRemoteCommit(t, env.remote, env.repoURL, "seed", map[string]string{"apps/demo/compose.yaml": remoteCompose})
+ writeBackupProjectFile(t, env.projectPath, ".env", "TOKEN=keep-me\n")
created, err := env.service.CreateSync(ctx, "0", gitops.CreateSyncRequest{
Name: "deploy-existing",
@@ -1653,7 +1646,7 @@ func TestGitOpsDeploy_LinksExistingProject(t *testing.T) {
}
func TestGitOpsImport_ForwardsDeployAndLifecycleFields(t *testing.T) {
- env := setupGitOpsBackupTestServiceInternal(t)
+ env := setupGitOpsBackupTestService(t)
ctx := t.Context()
require.NoError(t, env.service.settingsService.SetStringSetting(ctx, "lifecycleEnabled", "true"))
diff --git a/backend/internal/gitrepo/handler.go b/backend/internal/gitrepo/handler.go
index 515262ab6..38ffad320 100644
--- a/backend/internal/gitrepo/handler.go
+++ b/backend/internal/gitrepo/handler.go
@@ -165,14 +165,7 @@ func (h *GitRepositoryHandler) DeleteRepository(ctx context.Context, input *Dele
return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to delete git repository")
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Repository deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Repository deleted successfully", ""), nil
}
// TestRepository tests connectivity and authentication to a git repository.
@@ -183,14 +176,7 @@ func (h *GitRepositoryHandler) TestRepository(ctx context.Context, input *TestGi
return nil, huma.Error400BadRequest("Failed to test git repository connection: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Connection successful",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Connection successful", ""), nil
}
// ListBranches returns all branches from a git repository.
@@ -236,12 +222,5 @@ func (h *GitRepositoryHandler) SyncRepositories(ctx context.Context, input *Sync
return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to sync git repositories: "+err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Repositories synced successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Repositories synced successfully", ""), nil
}
diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.go
index 767153be2..226449fd7 100644
--- a/backend/internal/image/children/patch/service.go
+++ b/backend/internal/image/children/patch/service.go
@@ -11,6 +11,7 @@ import (
"maps"
"os"
"path/filepath"
+ "slices"
"strings"
"time"
@@ -46,6 +47,19 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/logging"
)
+// scheduledPatchUser is the actor recorded for scheduled patches and their verification scans.
+var (
+ scheduledPatchUser = usertypes.Actor{Username: "System"}
+ // errPatchInterrupted fails a patch cut off mid-run; its record keeps the message so it never reruns.
+ errPatchInterrupted = errors.New("image patch was interrupted before its result was recorded; start a new patch")
+)
+
+// Target is one image a scheduled patch run covers.
+type Target struct {
+ ImageID string `json:"imageId"`
+ ImageName string `json:"imageName"`
+}
+
// Service patches image OS packages in place using the Copacetic
// library and records each run in the image_patches table.
type Service struct {
@@ -98,128 +112,186 @@ func NewService(
// PatchImage starts a background patch run for the given image and returns the
// pending record (carrying the activity ID) immediately.
func (s *Service) PatchImage(ctx context.Context, envID, imageID string, opts imagepatch.PatchOptions, user usertypes.Actor) (*imagepatch.PatchRecord, error) {
- if strings.TrimSpace(opts.ScanID) != "" {
+ if opts.ScanID != "" {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
return nil, err
}
}
- runCtx := utils.ActivityRuntimeContext(ctx, nil)
+ record, run, err := s.startPatch(utils.ActivityRuntimeContext(ctx, nil), envID, imageID, opts, user, "")
+ if err != nil {
+ return nil, err
+ }
+ go func() { _ = run() }()
+ dto := record.ToDto()
+ return &dto, nil
+}
+
+// PatchTarget patches one scanned image and waits for the result. recordID keys
+// the patch so a redelivered task resumes its own record instead of patching twice.
+func (s *Service) PatchTarget(ctx context.Context, envID string, target Target, recordID string) error {
+ _, run, err := s.startPatch(ctx, envID, target.ImageID, imagepatch.PatchOptions{ScanID: target.ImageID}, scheduledPatchUser, recordID)
+ if err != nil || run == nil {
+ return err
+ }
+ return run()
+}
+
+// startPatch validates an image, records its patch as running, and returns the
+// function that performs it. A completed record under recordID returns no function.
+func (s *Service) startPatch(
+ ctx context.Context,
+ envID, imageID string,
+ opts imagepatch.PatchOptions,
+ user usertypes.Actor,
+ recordID string,
+) (*ImagePatchRecord, func() error, error) {
+ var existing ImagePatchRecord
+ if recordID != "" {
+ if err := s.db.WithContext(ctx).Where("id = ?", recordID).Limit(1).Find(&existing).Error; err != nil {
+ return nil, nil, fmt.Errorf("failed to load image patch record: %w", err)
+ }
+ if existing.Status == string(imagepatch.PatchStatusCompleted) {
+ return &existing, nil, nil
+ }
+ // A record still patching was cut off mid-run, so whether Copacetic already replaced its tag is unknown.
+ // Its failure keeps that message, so no later delivery runs the patch again either.
+ if existing.Status == string(imagepatch.PatchStatusPatching) || mo.PointerToOption(existing.Error).OrEmpty() == errPatchInterrupted.Error() {
+ s.finishPatchRecord(ctx, &existing, imagepatch.PatchStatusFailed, errPatchInterrupted.Error(), nil, 0)
+ return nil, nil, errPatchInterrupted
+ }
+ }
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- if requireContainerdImageStoreErr := requireContainerdImageStoreInternal(runCtx, dockerClient); requireContainerdImageStoreErr != nil {
- return nil, requireContainerdImageStoreErr
+ if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {
+ return nil, nil, storeErr
}
-
- imageInspect, err := dockerClient.ImageInspect(runCtx, imageID)
+ imageInspect, err := dockerClient.ImageInspect(ctx, imageID)
if err != nil {
- return nil, fmt.Errorf("failed to inspect image: %w", err)
+ return nil, nil, fmt.Errorf("failed to inspect image: %w", err)
}
if len(imageInspect.RepoTags) == 0 {
- return nil, common.ErrImageUntagged
+ return nil, nil, common.ErrImageUntagged
}
imageRef := imageInspect.RepoTags[0]
-
+ if existing.ID != "" {
+ // A replay patches the source its record names, so the output matches the saved PatchedRef.
+ if !slices.Contains(imageInspect.RepoTags, existing.OriginalRef) {
+ return nil, nil, fmt.Errorf("image is no longer tagged %s; start a new patch", existing.OriginalRef)
+ }
+ imageRef = existing.OriginalRef
+ }
// Never pulled or pushed: BuildKit cannot fetch the image to patch it.
if len(imageInspect.RepoDigests) == 0 {
- return nil, common.ErrImageLocalOnly
+ return nil, nil, common.ErrImageLocalOnly
}
- originalDigest := imageInspect.RepoDigests[0]
- // Resolve the stored scan report when patching from a scan.
mode := imagepatch.PatchModeUpdateAll
var reportData []byte
- if strings.TrimSpace(opts.ScanID) != "" {
- // Scan records are keyed by image ID; refuse a scan belonging to a
- // different image than the one selected by the route.
+ if opts.ScanID != "" {
+ // Scan records are keyed by image ID; refuse a scan of a different image than the route selected.
if opts.ScanID != imageInspect.ID {
- return nil, common.ErrPatchScanImageMismatch
+ return nil, nil, common.ErrPatchScanImageMismatch
}
var report vulnerability.VulnerabilityReportRecord
- if loadScanReportErr := s.db.WithContext(runCtx).First(&report, "image_id = ?", opts.ScanID).Error; loadScanReportErr != nil || report.Data == "" {
- return nil, common.ErrPatchScanReportUnavailable
+ if reportErr := s.db.WithContext(ctx).First(&report, "image_id = ?", opts.ScanID).Error; reportErr != nil || report.Data == "" {
+ return nil, nil, common.ErrPatchScanReportUnavailable
}
reportData = []byte(report.Data)
mode = imagepatch.PatchModeReport
}
- suffix := strings.TrimSpace(opts.Suffix)
- if suffix == "" {
- suffix = strings.TrimSpace(s.settingsService.GetSettingsConfig().ImagePatchSuffix.Value)
- }
- suffix = cmp.Or(suffix, "patched")
+ suffix := cmp.Or(strings.TrimSpace(opts.Suffix), strings.TrimSpace(s.settingsService.GetSettingsConfig().ImagePatchSuffix.Value), "patched")
patchedRef, err := resolvePatchedRef(imageRef, opts.PatchedTag, suffix)
if err != nil {
- return nil, err
+ return nil, nil, err
}
- // Copa reads registry credentials from the docker config; refresh it before
- // any registry lookups below.
- s.writeRegistryAuthConfigInternal(runCtx)
+ // Copa reads registry credentials from the docker config; refresh it before any registry lookups.
+ s.writeRegistryAuthConfig(ctx)
- // Unless the all-platforms setting is enabled, pin the patch target to the
- // platform-specific manifest digest matching this image so copa patches a
- // single platform and keeps the plain patched tag. Report-mode patches are
- // always single-platform in copa, so no pinning is needed there.
+ // Unless the all-platforms setting is on, pin update-all patches to this image's platform manifest so copa
+ // patches one platform and keeps the plain patched tag. Report-mode patches are always single-platform.
copaImageRef := imageRef
if mode == imagepatch.PatchModeUpdateAll && !s.settingsService.GetSettingsConfig().ImagePatchAllPlatforms.IsTrue() {
- if pinned := platformPinnedRefInternal(runCtx, imageRef, v1.Platform{
+ copaImageRef = cmp.Or(platformPinnedRef(ctx, imageRef, v1.Platform{
OS: imageInspect.Os,
Architecture: imageInspect.Architecture,
Variant: imageInspect.Variant,
- }); pinned != "" {
- copaImageRef = pinned
- }
- }
-
- activityID := s.startPatchActivityInternal(runCtx, envID, imageID, imageRef, &user)
- runCtx = s.activityService.Track(runCtx, activityID)
-
- record := &ImagePatchRecord{
- EnvironmentID: envID,
- OriginalImageID: imageInspect.ID,
- OriginalRef: imageRef,
- OriginalDigest: originalDigest,
- PatchedRef: patchedRef,
- Mode: string(mode),
- Status: string(imagepatch.PatchStatusPatching),
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
+ }), imageRef)
+ }
+
+ activityID := ""
+ if s.activityService != nil {
+ // Each attempt gets its own activity, since a finished activity never reopens.
+ started, startErr := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
+ EnvironmentID: envID,
+ Type: activitytypes.TypeImagePatch,
+ ResourceType: new("image"),
+ ResourceID: &imageID,
+ ResourceName: &imageRef,
+ StartedBy: &user,
+ Progress: new(0),
+ LatestMessage: "Image patch queued",
+ })
+ if startErr != nil {
+ slog.WarnContext(ctx, "failed to create image patch activity", "error", startErr, "imageRef", imageRef)
+ } else {
+ activityID = started.ID
+ }
+ }
+ runCtx := s.activityService.Track(ctx, activityID)
+
+ record := &existing
+ var saveErr error
+ if record.ID == "" {
+ record = &ImagePatchRecord{
+ EnvironmentID: envID,
+ OriginalImageID: imageInspect.ID,
+ OriginalRef: imageRef,
+ OriginalDigest: imageInspect.RepoDigests[0],
+ PatchedRef: patchedRef,
+ Mode: string(mode),
+ Status: string(imagepatch.PatchStatusPatching),
+ ActivityID: mo.EmptyableToOption(activityID).ToPointer(),
+ }
+ record.ID = recordID
+ saveErr = s.db.WithContext(ctx).Create(record).Error
+ } else {
+ // A retried record runs again under this attempt's activity, without the earlier attempt's error.
+ record.Status, record.Error, record.ActivityID = string(imagepatch.PatchStatusPatching), nil, mo.EmptyableToOption(activityID).ToPointer()
+ saveErr = s.db.WithContext(ctx).Model(record).Select("status", "error", "activity_id").Updates(record).Error
}
- if createPatchRecordErr := s.db.WithContext(runCtx).Create(record).Error; createPatchRecordErr != nil {
- return nil, fmt.Errorf("failed to create image patch record: %w", createPatchRecordErr)
+ if saveErr != nil {
+ // No worker will run, so settle the activity and release its cancel registration here.
+ saveErr = fmt.Errorf("failed to save image patch record: %w", saveErr)
+ s.completePatchActivity(runCtx, activityID, false, saveErr.Error())
+ return nil, nil, saveErr
}
- slog.InfoContext(runCtx, "image patch queued",
- "environmentId", envID,
- "imageRef", imageRef,
- "patchedRef", patchedRef,
- "mode", mode,
- "patchTarget", copaImageRef,
- )
-
- go s.patchInBackgroundInternal(runCtx, record, opts, reportData, copaImageRef, activityID)
-
- dto := record.ToDto()
- return &dto, nil
+ slog.InfoContext(ctx, "image patch queued", "environmentId", envID, "imageRef", imageRef, "patchedRef", patchedRef, "mode", mode, "patchTarget", copaImageRef)
+ return record, func() error { return s.runPatch(runCtx, record, opts, reportData, copaImageRef, activityID) }, nil
}
-func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) {
+// runPatch executes one recorded patch with Copacetic and settles its record and activity.
+func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) error {
+ fail := func(err error, durationMs int64) error {
+ s.finishPatchRecord(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, durationMs)
+ s.completePatchActivity(ctx, activityID, false, err.Error())
+ return err
+ }
select {
case s.patchSlot <- struct{}{}:
defer func() { <-s.patchSlot }()
case <-ctx.Done():
- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, ctx.Err().Error(), nil, 0)
- s.completePatchActivityInternal(ctx, activityID, false, ctx.Err().Error())
- return
+ return fail(ctx.Err(), 0)
}
if record.Mode == string(imagepatch.PatchModeReport) {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, 0)
- s.completePatchActivityInternal(ctx, activityID, false, err.Error())
- return
+ return fail(err, 0)
}
}
@@ -238,9 +310,7 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa
}()
}
if err != nil {
- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, 0)
- s.completePatchActivityInternal(ctx, activityID, false, err.Error())
- return
+ return fail(err, 0)
}
}
@@ -286,7 +356,7 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa
// Mirror copa's log output into the activity while the patch runs; BuildKit
// step progress stays quiet until copa exposes a progress writer upstream.
- s.appendPatchActivityInternal(ctx, activityID, 30, "Patching image packages via BuildKit")
+ s.appendPatchActivity(ctx, activityID, 30, "Patching image packages via BuildKit")
patchOut := activitylib.NewWriter(ctx, s.activityService, activityID, nil, "Patching image")
removeMirror := logging.AddLogrusMirror(patchOut)
patchErr := patch.Patch(ctx, copaOpts)
@@ -301,15 +371,12 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa
if strings.Contains(patchErr.Error(), `exporter "docker" could not be found`) {
patchErr = common.ErrPatchRequiresContainerdImageStore
}
- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, patchErr.Error(), nil, durationMs)
- slog.WarnContext(ctx, "image patch failed",
- "environmentId", record.EnvironmentID,
- "imageRef", record.OriginalRef,
- "durationMs", durationMs,
- "error", patchErr,
- )
- s.completePatchActivityInternal(ctx, activityID, false, patchErr.Error())
- return
+ // Interrupted mid-run, Copacetic may already have replaced the tag, so the record must never rerun.
+ if ctx.Err() != nil {
+ patchErr = errPatchInterrupted
+ }
+ slog.WarnContext(ctx, "image patch failed", "environmentId", record.EnvironmentID, "imageRef", record.OriginalRef, "durationMs", durationMs, "error", patchErr)
+ return fail(patchErr, durationMs)
}
// Count VEX statements as the number of patched packages, when available.
@@ -326,42 +393,37 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa
}
}
- s.verifyPatchedImageInternal(ctx, record, activityID)
+ s.verifyPatchedImage(ctx, record, activityID)
- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusCompleted, "", packagesUpdated, durationMs)
- slog.InfoContext(ctx, "image patch completed",
- "environmentId", record.EnvironmentID,
- "imageRef", record.OriginalRef,
- "patchedRef", record.PatchedRef,
- "durationMs", durationMs,
- )
- s.completePatchActivityInternal(ctx, activityID, true, "")
+ s.finishPatchRecord(ctx, record, imagepatch.PatchStatusCompleted, "", packagesUpdated, durationMs)
+ slog.InfoContext(ctx, "image patch completed", "environmentId", record.EnvironmentID, "imageRef", record.OriginalRef, "patchedRef", record.PatchedRef, "durationMs", durationMs)
+ s.completePatchActivity(ctx, activityID, true, "")
+ return nil
}
-func (s *Service) verifyPatchedImageInternal(ctx context.Context, record *ImagePatchRecord, activityID string) {
- // Warn when the expected patched tag is missing from the daemon (e.g. copa
- // took the multi-platform path and arch-suffixed the tag). When it exists,
- // re-scan it so the security page can show whether the patch worked.
+// verifyPatchedImage warns when the expected tag is missing (copa may arch-suffix
+// multi-platform tags) and otherwise re-scans it to show whether the patch worked.
+func (s *Service) verifyPatchedImage(ctx context.Context, record *ImagePatchRecord, activityID string) {
if dockerClient, err := s.dockerService.GetClient(ctx); err == nil {
- if patchedInspect, imageInspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); imageInspectErr != nil {
- slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", imageInspectErr)
- s.appendPatchActivityInternal(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")
+ if patchedInspect, inspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); inspectErr != nil {
+ slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)
+ s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")
} else if s.vulnerabilityService != nil && s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
- s.appendPatchActivityInternal(ctx, activityID, 95, "Re-scanning patched image to verify the patch")
- if _, scanImageErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, usertypes.Actor{Username: "System"}); scanImageErr != nil {
- slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanImageErr)
+ s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")
+ if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {
+ slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)
}
}
}
}
-// platformPinnedRefInternal resolves a tag reference to the manifest digest of
+// platformPinnedRef resolves a tag reference to the manifest digest of
// the given platform when the registry serves a multi-platform index. This
// makes copa patch a single platform (keeping the plain patched tag) instead
// of every platform in the index. Returns "" when the reference is not a
// multi-platform index or cannot be resolved, in which case the plain tag is
// used and copa's own discovery decides.
-func platformPinnedRefInternal(ctx context.Context, imageRef string, target v1.Platform) string {
+func platformPinnedRef(ctx context.Context, imageRef string, target v1.Platform) string {
ref, err := name.ParseReference(imageRef)
if err != nil {
return ""
@@ -411,11 +473,11 @@ func resolvePatchedRef(imageRef, patchedTag, suffix string) (string, error) {
return localName + ":" + tag, nil
}
-// writeRegistryAuthConfigInternal merges Arcane's registry credentials into the
+// writeRegistryAuthConfig merges Arcane's registry credentials into the
// docker config file copa reads for BuildKit registry auth. It only touches the
// config when DOCKER_CONFIG is set, i.e. the deployment (normally Arcane's own
// startup) owns that directory — a user's ~/.docker/config.json is never modified.
-func (s *Service) writeRegistryAuthConfigInternal(ctx context.Context) {
+func (s *Service) writeRegistryAuthConfig(ctx context.Context) {
configDir := strings.TrimSpace(os.Getenv("DOCKER_CONFIG"))
if configDir == "" || s.registryService == nil {
return
@@ -473,7 +535,7 @@ func (s *Service) writeRegistryAuthConfigInternal(ctx context.Context) {
}
}
-func (s *Service) finishPatchRecordInternal(ctx context.Context, record *ImagePatchRecord, status imagepatch.PatchStatus, errMessage string, packagesUpdated *int, durationMs int64) {
+func (s *Service) finishPatchRecord(ctx context.Context, record *ImagePatchRecord, status imagepatch.PatchStatus, errMessage string, packagesUpdated *int, durationMs int64) {
updates := map[string]any{
"status": string(status),
"duration_ms": durationMs,
@@ -517,8 +579,8 @@ func (s *Service) ListPatches(ctx context.Context, envID string, params paginati
return dtos, paginationResp, nil
}
-// PatchedRefs returns the set of patched image references recorded for an
-// environment, used to keep patch outputs from being patched again.
+// PatchedRefs returns every name a completed patch output can be scanned under
+// (normalized and familiar), used to keep patch outputs from being patched again.
func (s *Service) PatchedRefs(ctx context.Context, envID string) (map[string]struct{}, error) {
var refs []string
if err := s.db.WithContext(ctx).
@@ -528,9 +590,12 @@ func (s *Service) PatchedRefs(ctx context.Context, envID string) (map[string]str
Pluck("patched_ref", &refs).Error; err != nil {
return nil, fmt.Errorf("failed to list patched image refs: %w", err)
}
- set := make(map[string]struct{}, len(refs))
+ set := make(map[string]struct{}, len(refs)*2)
for _, ref := range refs {
set[ref] = struct{}{}
+ if named, err := reference.ParseNormalizedNamed(ref); err == nil {
+ set[reference.FamiliarString(named)] = struct{}{}
+ }
}
return set, nil
}
@@ -547,13 +612,7 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pag
if err != nil {
return nil, pagination.Response{}, err
}
- excludedNames := make([]string, 0, len(patchedRefs)*2)
- for ref := range patchedRefs {
- excludedNames = append(excludedNames, ref)
- if named, parseNormalizedNamedErr := reference.ParseNormalizedNamed(ref); parseNormalizedNamedErr == nil {
- excludedNames = append(excludedNames, reference.FamiliarString(named))
- }
- }
+ excludedNames := slices.Collect(maps.Keys(patchedRefs))
// Only list images that are actionable (fixable vulnerabilities) or carry
// patch history.
@@ -599,7 +658,7 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pag
for i := range scans {
imageIDs = append(imageIDs, scans[i].ID)
}
- lastPatchByImageID, lastPatchScanByImageID, err := s.latestPatchesByImageInternal(ctx, envID, imageIDs)
+ lastPatchByImageID, lastPatchScanByImageID, err := s.latestPatchesByImage(ctx, envID, imageIDs)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -636,10 +695,10 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pag
return targets, paginationResp, nil
}
-// latestPatchesByImageInternal loads the most recent patch run per original
+// latestPatchesByImage loads the most recent patch run per original
// image in two batched queries, plus the scan of each completed patch's output
// image so the original's row can show whether the patch worked.
-func (s *Service) latestPatchesByImageInternal(ctx context.Context, envID string, imageIDs []string) (map[string]*ImagePatchRecord, map[string]*imagepatch.PatchScanSummary, error) {
+func (s *Service) latestPatchesByImage(ctx context.Context, envID string, imageIDs []string) (map[string]*ImagePatchRecord, map[string]*imagepatch.PatchScanSummary, error) {
var patchRows []ImagePatchRecord
if err := s.db.WithContext(ctx).
Where("environment_id = ? AND original_image_id IN ?", envID, imageIDs).
@@ -704,75 +763,54 @@ func (s *Service) latestPatchesByImageInternal(ctx context.Context, envID string
return lastPatchByImageID, lastPatchScanByImageID, nil
}
-// PatchFlaggedImages patches every image whose latest completed vulnerability
-// scan found fixable vulnerabilities and stored a raw report. Images that are
-// themselves patch outputs are skipped (never patch a patched tag), as are
-// images already patched since their latest scan. Used by the scheduled
-// auto-patch job.
-func (s *Service) PatchFlaggedImages(ctx context.Context, envID string, user usertypes.Actor) (patched, skipped int, err error) {
- if requireFeatureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); requireFeatureErr != nil {
- return 0, 0, requireFeatureErr
+// Targets lists images whose latest completed scan found fixable vulnerabilities
+// and kept a report, skipping patch outputs and images patched since that scan.
+func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {
+ if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
+ return nil, err
}
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return 0, 0, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- if requireContainerdImageStoreErr := requireContainerdImageStoreInternal(ctx, dockerClient); requireContainerdImageStoreErr != nil {
- return 0, 0, requireContainerdImageStoreErr
+ if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {
+ return nil, storeErr
}
var scans []vulnerability.VulnerabilityScanRecord
- if loadFixableScansErr := s.db.WithContext(ctx).
+ if scansErr := s.db.WithContext(ctx).
Where("status = ? AND fixable_count > 0", vulnerability.ScanStatusCompleted).
Where("id IN (?)", s.db.Model(&vulnerability.VulnerabilityReportRecord{}).Select("image_id")).
Where("image_name NOT LIKE 'sha256:%' AND image_name NOT LIKE '%<none>%' AND image_name <> id").
- Find(&scans).Error; loadFixableScansErr != nil {
- return 0, 0, fmt.Errorf("failed to list vulnerability scans: %w", loadFixableScansErr)
+ Find(&scans).Error; scansErr != nil {
+ return nil, fmt.Errorf("failed to list vulnerability scans: %w", scansErr)
}
-
patchedRefs, err := s.PatchedRefs(ctx, envID)
if err != nil {
- return 0, 0, err
+ return nil, err
}
+ targets := make([]Target, 0, len(scans))
for i := range scans {
- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
- return patched, skipped + len(scans) - i, fmt.Errorf("remaining patches stopped because feature %s was disabled: %w", features.VulnerabilityManagement, common.ErrFeatureDisabled)
- }
scan := &scans[i]
-
if _, isPatchOutput := patchedRefs[scan.ImageName]; isPatchOutput {
- skipped++
continue
}
-
- // Skip images already patched (or being patched) since their latest scan.
+ // Skip images already patched, or being patched, since their latest scan.
var recent int64
- if countRecentPatchesErr := s.db.WithContext(ctx).
+ if countErr := s.db.WithContext(ctx).
Model(&ImagePatchRecord{}).
Where("environment_id = ? AND original_image_id = ? AND status IN ? AND created_at >= ?",
envID, scan.ID, []string{string(imagepatch.PatchStatusCompleted), string(imagepatch.PatchStatusPatching)}, scan.ScanTime).
- Count(&recent).Error; countRecentPatchesErr == nil && recent > 0 {
- skipped++
- continue
- }
-
- if _, patchImageErr := s.PatchImage(ctx, envID, scan.ID, imagepatch.PatchOptions{ScanID: scan.ID}, user); patchImageErr != nil {
- slog.WarnContext(ctx, "scheduled image patch failed to start",
- "imageId", scan.ID,
- "imageName", scan.ImageName,
- "error", patchImageErr,
- )
- skipped++
+ Count(&recent).Error; countErr == nil && recent > 0 {
continue
}
- patched++
+ targets = append(targets, Target{ImageID: scan.ID, ImageName: scan.ImageName})
}
-
- return patched, skipped, nil
+ return targets, nil
}
// Copa needs BuildKit's docker exporter, which dockerd only offers with the containerd image store.
-func requireContainerdImageStoreInternal(ctx context.Context, dockerClient *client.Client) error {
+func requireContainerdImageStore(ctx context.Context, dockerClient *client.Client) error {
info, err := dockerClient.Info(ctx, client.InfoOptions{})
if err != nil {
return fmt.Errorf("failed to inspect Docker: %w", err)
@@ -785,28 +823,7 @@ func requireContainerdImageStoreInternal(ctx context.Context, dockerClient *clie
return common.ErrPatchRequiresContainerdImageStore
}
-func (s *Service) startPatchActivityInternal(ctx context.Context, envID, imageID, imageRef string, user *usertypes.Actor) string {
- if s.activityService == nil {
- return ""
- }
- started, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
- EnvironmentID: envID,
- Type: activitytypes.TypeImagePatch,
- ResourceType: new("image"),
- ResourceID: &imageID,
- ResourceName: &imageRef,
- StartedBy: user,
- Progress: new(0),
- LatestMessage: "Image patch queued",
- })
- if err != nil {
- slog.WarnContext(ctx, "failed to create image patch activity", "error", err, "imageRef", imageRef)
- return ""
- }
- return started.ID
-}
-
-func (s *Service) appendPatchActivityInternal(ctx context.Context, activityID string, progress int, message string) {
+func (s *Service) appendPatchActivity(ctx context.Context, activityID string, progress int, message string) {
if s.activityService == nil || activityID == "" {
return
}
@@ -819,7 +836,7 @@ func (s *Service) appendPatchActivityInternal(ctx context.Context, activityID st
}
}
-func (s *Service) completePatchActivityInternal(ctx context.Context, activityID string, success bool, errMessage string) {
+func (s *Service) completePatchActivity(ctx context.Context, activityID string, success bool, errMessage string) {
if s.activityService == nil || activityID == "" {
return
}
@@ -827,18 +844,13 @@ func (s *Service) completePatchActivityInternal(ctx context.Context, activityID
status := activitytypes.StatusSuccess
message := "Image patch completed"
var errorPtr *string
- if !success {
- if activitylib.CancelledByContext(ctx) {
- status = activitytypes.StatusCancelled
- message = "Image patch cancelled"
- } else {
- status = activitytypes.StatusFailed
- message = "Image patch failed"
- if strings.TrimSpace(errMessage) != "" {
- errorPtr = &errMessage
- message = errMessage
- }
- }
+ switch {
+ case success:
+ case activitylib.CancelledByContext(ctx):
+ status, message = activitytypes.StatusCancelled, "Image patch cancelled"
+ default:
+ status, message = activitytypes.StatusFailed, cmp.Or(errMessage, "Image patch failed")
+ errorPtr = mo.EmptyableToOption(errMessage).ToPointer()
}
if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errorPtr); err != nil {
diff --git a/backend/internal/image/children/patch/service_test.go b/backend/internal/image/children/patch/service_test.go
index 55b6c2a28..c22a378a4 100644
--- a/backend/internal/image/children/patch/service_test.go
+++ b/backend/internal/image/children/patch/service_test.go
@@ -95,7 +95,7 @@ func TestListPatchTargets_ExcludesUntaggedImages(t *testing.T) {
require.Equal(t, "nginx:latest", targets[0].ImageRef)
}
-func patchFeatureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {
+func patchFeatureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {
t.Helper()
gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -110,7 +110,7 @@ func patchFeatureSettingsInternal(t *testing.T) (*settings.SettingsService, *dat
}
func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {
- settingsSvc, db := patchFeatureSettingsInternal(t)
+ settingsSvc, db := patchFeatureSettings(t)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
svc := &Service{
settingsService: settingsSvc,
@@ -121,7 +121,7 @@ func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {
require.ErrorIs(t, err, common.ErrFeatureDisabled)
_, _, err = svc.ListPatchTargets(t.Context(), "0", pagination.QueryParams{})
require.ErrorIs(t, err, common.ErrFeatureDisabled)
- _, _, err = svc.PatchFlaggedImages(t.Context(), "0", user.Actor{})
+ _, err = svc.Targets(t.Context(), "0")
require.ErrorIs(t, err, common.ErrFeatureDisabled)
// Standalone patching still reaches Docker; this test deliberately has no daemon.
_, err = svc.PatchImage(t.Context(), "0", "test-image", imagepatch.PatchOptions{}, user.Actor{})
@@ -131,7 +131,7 @@ func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {
}
func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {
- settingsSvc, db := patchFeatureSettingsInternal(t)
+ settingsSvc, db := patchFeatureSettings(t)
svc := &Service{settingsService: settingsSvc, db: db, patchSlot: make(chan struct{}, 1)}
record := &ImagePatchRecord{EnvironmentID: "0", OriginalImageID: "test-image", Mode: string(imagepatch.PatchModeReport), Status: string(imagepatch.PatchStatusPatching)}
require.NoError(t, db.Create(record).Error)
@@ -139,7 +139,7 @@ func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {
done := make(chan struct{})
go func() {
defer close(done)
- svc.patchInBackgroundInternal(t.Context(), record, imagepatch.PatchOptions{ScanID: "test-image"}, nil, "", "")
+ _ = svc.runPatch(t.Context(), record, imagepatch.PatchOptions{ScanID: "test-image"}, nil, "", "")
}()
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
// The active patch owns its slot until it finishes normally.
@@ -159,7 +159,7 @@ func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {
}
func TestDisabledFeatureSkipsPatchVerificationScan(t *testing.T) {
- settingsSvc, _ := patchFeatureSettingsInternal(t)
+ settingsSvc, _ := patchFeatureSettings(t)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
var inspected atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -178,7 +178,7 @@ func TestDisabledFeatureSkipsPatchVerificationScan(t *testing.T) {
vulnerabilityService: &vulnerability.VulnerabilityService{},
}
require.NotPanics(t, func() {
- svc.verifyPatchedImageInternal(t.Context(), &ImagePatchRecord{EnvironmentID: "0", PatchedRef: "test:patched"}, "")
+ svc.verifyPatchedImage(t.Context(), &ImagePatchRecord{EnvironmentID: "0", PatchedRef: "test:patched"}, "")
})
require.Equal(t, int32(1), inspected.Load())
}
diff --git a/backend/internal/image/handler.go b/backend/internal/image/handler.go
index a20faacda..571115ca5 100644
--- a/backend/internal/image/handler.go
+++ b/backend/internal/image/handler.go
@@ -16,7 +16,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/system"
uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload"
"go.getarcane.app/builds/types"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -203,12 +203,7 @@ func (h *ImageHandler) TagImage(ctx context.Context, input *TagImageInput) (*han
return nil, huma.Error500InternalServerError(fmt.Sprintf("failed to tag image: %v", tagImageErr))
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Image tagged successfully"},
- },
- }, nil
+ return handlerutil.MessageOutput("Image tagged successfully", ""), nil
}
// GetImageHistory returns Docker image layer history.
@@ -304,14 +299,7 @@ func (h *ImageHandler) RemoveImage(ctx context.Context, input *RemoveImageInput)
return nil, huma.Error500InternalServerError("Failed to remove image: " + removeImageErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Image removed successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Image removed successfully", ""), nil
}
// PullImage pulls a Docker image with streaming progress.
diff --git a/backend/internal/image/service.go b/backend/internal/image/service.go
index 142967fcc..edddafcf2 100644
--- a/backend/internal/image/service.go
+++ b/backend/internal/image/service.go
@@ -13,11 +13,16 @@ import (
"time"
"github.com/distribution/reference"
+ "github.com/getarcaneapp/arcane/types/v2"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
+ "github.com/getarcaneapp/arcane/types/v2/features"
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/system"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
vulnerabilitytypes "github.com/getarcaneapp/arcane/types/v2/vulnerability"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
"github.com/moby/moby/client"
@@ -30,6 +35,7 @@ import (
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -39,12 +45,19 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
)
+// patchSlot is one patch task's output; failures travel as task errors.
+type patchSlot struct {
+ Skipped bool `json:"skipped,omitempty"`
+}
+
type ImageService struct {
db *database.DB
dockerService *dockerInternal.DockerClientService
@@ -53,8 +66,10 @@ type ImageService struct {
vulnerabilityService *vulnerability.VulnerabilityService
eventService *event.EventService
- patch *patch.Service
- projectIDCache *hot.HotCache[struct{}, map[string]string]
+ settingsService *settings.SettingsService
+ patch *patch.Service
+ patchWorkflow *flow.Workflow
+ projectIDCache *hot.HotCache[struct{}, map[string]string]
}
func NewImageService(
@@ -74,6 +89,7 @@ func NewImageService(
imageUpdateService: imageUpdateService,
vulnerabilityService: vulnerabilityService,
eventService: eventService,
+ settingsService: settingsService,
patch: patch.NewService(db, dockerService, settingsService, activityService, registryService, vulnerabilityService),
projectIDCache: hot.NewHotCache[struct{}, map[string]string](hot.LRU, 1).
WithTTL(projectIDCacheTTL).
@@ -81,11 +97,101 @@ func NewImageService(
}
}
-// PatchFlaggedImages patches eligible images for the scheduled patch job.
-func (s *ImageService) PatchFlaggedImages(ctx context.Context, environmentID string, user usertypes.Actor) (patched, skipped int, err error) {
- return s.patch.PatchFlaggedImages(ctx, environmentID, user)
+// RegisterWorkflows defines the scheduled auto-patch: find images with fixable
+// findings, patch each one in turn, then report. Call it before the host starts.
+func (s *ImageService) RegisterWorkflows(engine *flow.Engine) error {
+ patchWorkflow, err := engine.Define(flow.Definition{
+ Name: "auto-patch",
+ Version: 1,
+ Fingerprint: "e399c9c07ea15a7d6a39f086529110040a4d2cc95daf72c1464363c7f5bb5efc",
+ // Copacetic runs one BuildKit patch at a time and mirrors a process-wide logger.
+ Concurrency: 1,
+ Timeout: 12 * time.Hour,
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeImagePatch,
+ ResourceType: new("images"),
+ ResourceName: new("Scheduled image patch"),
+ Step: "Finding patchable images",
+ LatestMessage: "Scheduled image patch started",
+ },
+ Labels: map[string]string{
+ "discover": "Finding patchable images",
+ "patch": "Patching images",
+ "finalize": "Finishing image patch",
+ },
+ Steps: []workflow.StepSpec{
+ workflow.Step("discover", engine.Handler(func(ctx context.Context, _ flow.Task) (any, error) {
+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
+ return []patch.Target{}, nil
+ }
+ targets, err := s.patch.Targets(ctx, types.LocalDockerEnvironmentID)
+ if errors.Is(err, common.ErrPatchRequiresContainerdImageStore) {
+ return nil, err
+ }
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ return targets, nil
+ })),
+ workflow.ForEach("patch", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var target patch.Target
+ if err := t.DecodeItem(&target); err != nil {
+ return nil, err
+ }
+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
+ return patchSlot{Skipped: true}, nil
+ }
+ // The record ID is stable per task, so a redelivery resumes its own patch.
+ return nil, s.patch.PatchTarget(ctx, types.LocalDockerEnvironmentID, target, fmt.Sprintf("%s-%d", t.ActivityID(), t.Index()))
+ }),
+ workflow.WithItemsFrom("discover"),
+ workflow.WithMaxParallel(1),
+ workflow.WithFailurePolicy(workflow.TolerateFailures),
+ workflow.WithMaxAttempts(2),
+ workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),
+ workflow.Step("finalize", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var targets []patch.Target
+ if err := t.DecodeOutput("discover", &targets); err != nil {
+ return nil, err
+ }
+ slots, err := flow.Results[patchSlot](t, "patch")
+ if err != nil {
+ return nil, err
+ }
+ outcome := scheduler.Outcome{Status: scheduler.Succeeded}
+ patched, skipped := 0, 0
+ for index, slot := range slots {
+ switch {
+ case slot.Err != "":
+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ResourceType: "image", ID: targets[index].ImageName, Status: scheduler.Failed, Message: slot.Err})
+ case slot.Value.Skipped:
+ skipped++
+ default:
+ patched++
+ }
+ }
+ outcome.Message = fmt.Sprintf("Image patch completed: %d patched, %d failed", patched, len(outcome.Targets))
+ switch {
+ case skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)):
+ outcome.Status = kit.Ternary(patched > 0, scheduler.Partial, scheduler.Skipped)
+ outcome.Message = fmt.Sprintf("Remaining patches stopped because feature %s was disabled", features.VulnerabilityManagement)
+ case len(outcome.Targets) > 0:
+ outcome.Status = scheduler.Partial
+ }
+ return outcome, nil
+ }), workflow.WithInputFrom("discover")),
+ },
+ })
+ if err != nil {
+ return err
+ }
+ s.patchWorkflow = patchWorkflow
+ return nil
}
+// PatchWorkflow is the scheduled auto-patch workflow.
+func (s *ImageService) PatchWorkflow() *flow.Workflow { return s.patchWorkflow }
+
// newAttestationsServiceInternal builds the attestation reader from the image
// service's Docker client and registry credentials.
func (s *ImageService) newAttestationsServiceInternal() *attestations.Service {
diff --git a/backend/internal/image/service_test.go b/backend/internal/image/service_test.go
index cb2fa6cc8..d2423a797 100644
--- a/backend/internal/image/service_test.go
+++ b/backend/internal/image/service_test.go
@@ -35,6 +35,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
)
@@ -742,3 +743,10 @@ func TestImageServiceContainerTagUpdatesStayScoped(t *testing.T) {
require.NotContains(t, checks, "moving-no-id", "a missing runtime image ID matches nothing")
require.NotSame(t, checks["moving"], checks["moving-twin"], "each container receives its own copy")
}
+
+func TestPatchWorkflowDefinitions(t *testing.T) {
+ harness := flowtest.New(t, nil)
+ require.NoError(t, (&ImageService{}).RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
+}
diff --git a/backend/internal/imageupdate/handler.go b/backend/internal/imageupdate/handler.go
index e28802e3d..e02d2773c 100644
--- a/backend/internal/imageupdate/handler.go
+++ b/backend/internal/imageupdate/handler.go
@@ -99,7 +99,7 @@ func (h *ImageUpdateHandler) CheckMultipleImages(ctx context.Context, input *Che
}
runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
- results, err := h.imageUpdateService.CheckMultipleImages(runtimeCtx, input.Body.ImageRefs, input.Body.Credentials)
+ results, err := h.imageUpdateService.CheckImages(runtimeCtx, imageupdate.CheckRequest{ImageRefs: input.Body.ImageRefs}, input.Body.Credentials)
if err != nil {
return nil, huma.Error500InternalServerError("Failed to check image updates: " + err.Error())
}
@@ -114,7 +114,7 @@ func (h *ImageUpdateHandler) CheckMultipleImages(ctx context.Context, input *Che
func (h *ImageUpdateHandler) CheckAllImages(ctx context.Context, input *CheckAllImagesInput) (*handlerutil.Out[imageupdate.BatchResponse], error) {
runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
- results, err := h.imageUpdateService.CheckAllImages(runtimeCtx, 0, input.Body.Credentials)
+ results, err := h.imageUpdateService.CheckImages(runtimeCtx, imageupdate.CheckRequest{All: true}, input.Body.Credentials)
if err != nil {
return nil, huma.Error500InternalServerError("Failed to check all images: " + err.Error())
}
diff --git a/backend/internal/imageupdate/service.go b/backend/internal/imageupdate/service.go
index 044bf709c..b6aaa91e6 100644
--- a/backend/internal/imageupdate/service.go
+++ b/backend/internal/imageupdate/service.go
@@ -11,25 +11,29 @@ import (
"strings"
"sync"
"time"
+ "uuid"
"github.com/containerd/errdefs"
"github.com/distribution/reference"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
"github.com/moby/moby/client"
+ "github.com/samber/hot"
"github.com/samber/mo"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"go.getarcane.app/updater/digest"
"go.getarcane.app/updater/refs"
- "golang.org/x/sync/errgroup"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -42,11 +46,24 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
)
+const (
+ imageCheckConcurrency = 10
+ imageCheckAttempts = 3
+)
+
+var (
+ // errCheckInterrupted fails the images of a manual check that a restart interrupted.
+ errCheckInterrupted = errors.New("image check was interrupted by a restart; run it again")
+ imageCheckRetryBackoff = 30 * time.Second
+ imageCheckRetryBackoffMax = 2 * time.Minute
+)
+
type ImageUpdateService struct {
db *database.DB
settingsService *settings.SettingsService
@@ -57,7 +74,13 @@ type ImageUpdateService struct {
registryLimiter *ratelimit.RegistryRateLimiter
tags *tags.Service
activityService *activity.ActivityService
- notifyMu sync.Mutex
+ engine *flow.Engine
+ checkWorkflow *flow.Workflow
+ credentials *hot.HotCache[string, []containerregistry.Credential]
+ // requestCredentials holds credentials a caller supplied for one check, by its input's CredentialsKey,
+ // so they never reach the workflow journal. They expire with the check's timeout if nothing deletes them.
+ requestCredentials *hot.HotCache[string, []containerregistry.Credential]
+ notifyMu sync.Mutex
}
type ImageParts struct {
@@ -66,6 +89,36 @@ type ImageParts struct {
Tag string
}
+// imageCheckResultInternal is a check task's output; failures travel as errors.
+type imageCheckResultInternal struct {
+ Local bool `json:"local,omitempty"`
+ // Result is the ref's own result, kept when the caller reports results; tags sharing an image share its stored row.
+ Result *imageupdate.Response `json:"result,omitempty"`
+ // Failure is a reported check's error, which keeps its Result instead of failing the slot.
+ Failure string `json:"failure,omitempty"`
+}
+
+// imageCheckInput is an image-check payload; Report keeps every ref's result in the output for the caller.
+type imageCheckInput struct {
+ imageupdate.CheckRequest
+ Report bool `json:"report,omitempty"`
+ CredentialsKey string `json:"credentialsKey,omitempty"`
+}
+
+// imageCheckOutput is the image-check outcome plus, when reported, each ref's result and the current
+// container tag results, failed checks included.
+type imageCheckOutput struct {
+ scheduler.Outcome
+ Results imageupdate.BatchResponse `json:"results,omitempty"`
+ ContainerUpdates map[string]*imageupdate.Response `json:"containerUpdates,omitempty"`
+}
+
+// tagCheckOutput is the tags step's output: failed container checks and, when reported, every container's result.
+type tagCheckOutput struct {
+ Failures []scheduler.TargetOutcome `json:"failures"`
+ Updates map[string]*imageupdate.Response `json:"updates,omitempty"`
+}
+
type localImageSnapshot struct {
ImageID string
Repository string
@@ -96,6 +149,10 @@ func NewImageUpdateService(
registryLimiter: registryLimiter,
tags: tags.NewService(registryService, dockerService, settingsService, registryLimiter),
activityService: activityService,
+ credentials: hot.NewHotCache[string, []containerregistry.Credential](hot.LRU, 8).WithTTL(time.Minute).Build(),
+ // They outlive the longest check: its slot wait plus its workflow timeout.
+ requestCredentials: hot.NewHotCache[string, []containerregistry.Credential](hot.LRU, 64).
+ WithTTL(timeouts.DefaultActivitySlotWait + timeouts.DefaultImageUpdateScan).Build(),
}
}
@@ -177,21 +234,20 @@ func (s *ImageUpdateService) composeBuildImageRefsInternal(ctx context.Context)
return buildRefs, nil
}
-func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Context, resourceName string, count int) string {
+func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Context, imageRef string) string {
if s.activityService == nil {
return ""
}
- resourceType := kit.Ternary(count > 1, "images", "image")
localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
EnvironmentID: "0",
Type: activitytypes.TypeImageUpdateCheck,
Queue: true,
- ResourceType: &resourceType,
- ResourceName: mo.EmptyableToOption(strings.TrimSpace(resourceName)).ToPointer(),
+ ResourceType: new("image"),
+ ResourceName: mo.EmptyableToOption(strings.TrimSpace(imageRef)).ToPointer(),
Step: "Checking image updates",
LatestMessage: "Image update check started",
Metadata: database.JSON{
- "imageCount": count,
+ "imageCount": 1,
},
})
if err != nil {
@@ -201,7 +257,9 @@ func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Contex
return localActivity.ID
}
-func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx context.Context, activityID string, level activitytypes.MessageLevel, message string, progress int, step string) {
+// appendImageUpdateActivityMessageInternal appends a check message; a nil
+// progress leaves progress to the activity's owner, such as the workflow engine.
+func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx context.Context, activityID string, level activitytypes.MessageLevel, message string, progress *int, step string) {
if s.activityService == nil || activityID == "" || strings.TrimSpace(message) == "" {
return
}
@@ -209,7 +267,7 @@ func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx contex
if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{
Level: level,
Message: message,
- Progress: &progress,
+ Progress: progress,
Step: step,
}); err != nil {
slog.DebugContext(ctx, "failed to append image update activity message", "activityId", activityID, "error", err)
@@ -242,14 +300,14 @@ func (s *ImageUpdateService) completeImageUpdateActivityInternal(ctx context.Con
func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef string) (*imageupdate.Response, error) {
startTime := time.Now()
- activityID := s.startImageUpdateActivityInternal(ctx, imageRef, 1)
+ activityID := s.startImageUpdateActivityInternal(ctx, imageRef)
ctx = s.activityService.Track(ctx, activityID)
activitylib.AwaitHandlerActivitySlot(ctx, s.activityService, activityID, "0")
if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {
result.ResponseTimeMs = int(time.Since(startTime).Milliseconds())
result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 100)
+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, new(100))
if s.eventService != nil {
metadata := database.JSON{
"action": "check_update",
@@ -269,7 +327,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef stri
return result, nil
}
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, "Checking "+imageRef, 20, "Checking remote digest")
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, "Checking "+imageRef, new(20), "Checking remote digest")
parts := s.parseImageReference(imageRef)
if parts == nil {
@@ -333,7 +391,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef stri
digestResult.ResponseTimeMs = int(time.Since(startTime).Milliseconds())
digestResult.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
if digestResult.UpdateType == UpdateTypeLocal {
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — local build, registry check skipped", 100, "Skipping image update check")
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — local build, registry check skipped", new(100), "Skipping image update check")
}
metadata := database.JSON{
"action": "check_update",
@@ -985,29 +1043,6 @@ func buildImageUpdateRepositoryInternal(parts *ImageParts) string {
return fmt.Sprintf("%s/%s", strings.TrimSpace(parts.Registry), repository)
}
-func countBatchResultOutcomesInternal(imageRefs []string, results map[string]*imageupdate.Response) (int, int) {
- successCount := 0
- errorCount := 0
-
- for _, imageRef := range imageRefs {
- result := results[imageRef]
- hasError := result == nil
- if result != nil {
- hasError = strings.TrimSpace(result.Error) != ""
- for _, containerUpdate := range result.ContainerUpdates {
- hasError = hasError || containerUpdate.Error != ""
- }
- }
- if !hasError {
- successCount++
- continue
- }
- errorCount++
- }
-
- return successCount, errorCount
-}
-
// imageCheckResultMessageInternal derives an activity message level and text from
// a per-image update check result: errors become ERROR, available updates become
// SUCCESS, and up-to-date images stay INFO.
@@ -1367,70 +1402,27 @@ func (s *ImageUpdateService) MarkUpdatesAsNotified(ctx context.Context, imageIDs
Update("notification_sent", true).Error
}
-type batchImage struct {
- refs []string
- canonicalRef string
- parts *ImageParts
-}
-
-type batchImageProgressRecorder struct {
- mu sync.Mutex
- results map[string]*imageupdate.Response
- completed int
- total int
-}
-
-func (r *batchImageProgressRecorder) recordInternal(localRefs []string, res *imageupdate.Response) int {
- r.mu.Lock()
- defer r.mu.Unlock()
-
- r.completed++
- progress := 10 + int(float64(r.completed)/float64(r.total)*80)
- for _, imageRef := range localRefs {
- r.results[imageRef] = res
- }
- return progress
-}
-
-func (s *ImageUpdateService) parseAndGroupImagesInternal(imageRefs []string) (map[string]map[string]struct{}, map[string]*imageupdate.Response, []batchImage) {
- regRepos := make(map[string]map[string]struct{})
- results := make(map[string]*imageupdate.Response)
- var images []batchImage
+// groupImageRefsInternal dedupes refs naming the same registry/repo:tag into one
+// check; the first ref of a group is canonical. Digest-pinned and invalid refs
+// each form their own group.
+func (s *ImageUpdateService) groupImageRefsInternal(imageRefs []string) [][]string {
+ var groups [][]string
indexByNormalizedRef := make(map[string]int)
-
for _, imageRef := range imageRefs {
- if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {
- results[imageRef] = result
- continue
- }
-
parts := s.parseImageReference(imageRef)
- if parts == nil {
- results[imageRef] = &imageupdate.Response{
- Error: "Invalid image reference format",
- CheckTime: time.Now(),
- ResponseTimeMs: 0,
- }
+ if parts == nil || digestPinnedImageUpdateResultInternal(imageRef).IsPresent() {
+ groups = append(groups, []string{imageRef})
continue
}
- if _, ok := regRepos[parts.Registry]; !ok {
- regRepos[parts.Registry] = make(map[string]struct{})
- }
- regRepos[parts.Registry][parts.Repository] = struct{}{}
normalizedRef := strings.ToLower(fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag))
- if idx, exists := indexByNormalizedRef[normalizedRef]; exists {
- images[idx].refs = append(images[idx].refs, imageRef)
+ if index, exists := indexByNormalizedRef[normalizedRef]; exists {
+ groups[index] = append(groups[index], imageRef)
continue
}
-
- indexByNormalizedRef[normalizedRef] = len(images)
- images = append(images, batchImage{
- refs: []string{imageRef},
- canonicalRef: imageRef,
- parts: parts,
- })
+ indexByNormalizedRef[normalizedRef] = len(groups)
+ groups = append(groups, []string{imageRef})
}
- return regRepos, results, images
+ return groups
}
func (
@@ -1443,23 +1435,21 @@ func (
) (
*imageupdate.Response,
*localImageSnapshot,
+ error,
) {
if s.registryService == nil {
- return &imageupdate.Response{
- Error: "registry service unavailable",
- CheckTime: time.Now(),
- ResponseTimeMs: 0,
- }, nil
+ err := errors.New("registry service unavailable")
+ return &imageupdate.Response{Error: err.Error(), CheckTime: time.Now()}, nil, err
}
start := time.Now()
imageRef := fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag)
snapshot, ldErr := s.inspectLocalImageSnapshotInternal(ctx, imageRef, composeBuildRefs)
if ldErr == nil && snapshot.IsLocalBuild {
- return localBuildImageUpdateResultInternal(snapshot, int(time.Since(start).Milliseconds())), snapshot
+ return localBuildImageUpdateResultInternal(snapshot, int(time.Since(start).Milliseconds())), snapshot, nil
}
if ldErr != nil && errdefs.IsNotFound(ldErr) && isLocalBuildImageRefInternal(imageRef, composeBuildRefs) {
- return missingLocalBuildImageUpdateResultInternal(parts.Tag, int(time.Since(start).Milliseconds())), nil
+ return missingLocalBuildImageUpdateResultInternal(parts.Tag, int(time.Since(start).Milliseconds())), nil, nil
}
registryCtx, registryCancel := s.registryContextInternal(ctx)
@@ -1477,7 +1467,7 @@ func (
resp.AuthRegistry = digestResult.AuthRegistry
resp.UsedCredential = digestResult.UsedCredential
}
- return resp, nil
+ return resp, nil, digestErr
}
if ldErr != nil {
@@ -1497,7 +1487,7 @@ func (
AuthUsername: digestResult.AuthUsername,
AuthRegistry: digestResult.AuthRegistry,
UsedCredential: digestResult.UsedCredential,
- }, nil
+ }, nil, nil
}
return &imageupdate.Response{
Error: ldErr.Error(),
@@ -1507,7 +1497,7 @@ func (
AuthUsername: digestResult.AuthUsername,
AuthRegistry: digestResult.AuthRegistry,
UsedCredential: digestResult.UsedCredential,
- }, nil
+ }, nil, ldErr
}
}
@@ -1532,30 +1522,40 @@ func (
AuthUsername: digestResult.AuthUsername,
AuthRegistry: digestResult.AuthRegistry,
UsedCredential: digestResult.UsedCredential,
- }, snapshot
+ }, snapshot, nil
}
-func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context, externalCreds []containerregistry.Credential) []containerregistry.Credential {
- if len(externalCreds) > 0 {
- filtered := make([]containerregistry.Credential, 0, len(externalCreds))
- for _, cred := range externalCreds {
- if !cred.Enabled || strings.TrimSpace(cred.URL) == "" || strings.TrimSpace(cred.Username) == "" || strings.TrimSpace(cred.Token) == "" {
- continue
- }
- filtered = append(filtered, cred)
- }
- return filtered
+// resolveBatchCredentialsInternal returns the caller's credentials for the check, or else the enabled stored
+// registry credentials, cached per scan so its tasks decrypt them once while each new scan sees registry edits.
+// Failures are not cached. Caller credentials live only in memory, so a check resumed after a restart uses the stored ones.
+func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context, t flow.Task, input imageCheckInput) []containerregistry.Credential {
+ if supplied, ok, _ := s.requestCredentials.Get(input.CredentialsKey); ok && input.CredentialsKey != "" {
+ return supplied
}
-
if s.registryService == nil {
return nil
}
-
- registries, err := s.registryService.GetEnabledRegistries(ctx)
+ scanID := t.ActivityID()
+ credentials, _, err := s.credentials.GetWithLoaders(scanID, func([]string) (map[string][]containerregistry.Credential, error) {
+ loaded, loadErr := s.loadStoredCredentialsInternal(ctx)
+ if loadErr != nil {
+ return nil, loadErr
+ }
+ return map[string][]containerregistry.Credential{scanID: loaded}, nil
+ })
if err != nil {
slog.DebugContext(ctx, "failed to load enabled registries for batch check", "error", err.Error())
return nil
}
+ return credentials
+}
+
+// loadStoredCredentialsInternal decrypts the enabled stored registry credentials.
+func (s *ImageUpdateService) loadStoredCredentialsInternal(ctx context.Context) ([]containerregistry.Credential, error) {
+ registries, err := s.registryService.GetEnabledRegistries(ctx)
+ if err != nil {
+ return nil, err
+ }
credentials := make([]containerregistry.Credential, 0, len(registries))
for _, reg := range registries {
@@ -1581,215 +1581,351 @@ func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context
})
}
- return credentials
+ return credentials, nil
}
-func (
- s *ImageUpdateService,
-) checkBatchImageInternal(
- ctx context.Context,
- activityID string,
- resolvedCreds []containerregistry.Credential,
- img batchImage,
- composeBuildRefs map[string]struct{},
- recorder *batchImageProgressRecorder,
-) error {
- registryRecord := img.parts.Registry
-
- if err := s.registryLimiter.Acquire(ctx, registryRecord); err != nil {
- slog.DebugContext(ctx, "skipping image check: registry limiter acquire failed",
- "imageRef", img.canonicalRef,
- "registry", registryRecord,
- "error", err.Error())
- if ctx.Err() != nil {
- return ctx.Err()
- }
-
- res := &imageupdate.Response{
- Error: err.Error(),
- CheckTime: time.Now(),
- ResponseTimeMs: 0,
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- }
- s.recordBatchImageCheckInternal(ctx, activityID, img, res, nil, recorder)
- return nil
+// recordImageCheckInternal saves a check result and reports it on the activity.
+func (s *ImageUpdateService) recordImageCheckInternal(ctx context.Context, activityID, imageRef string, res *imageupdate.Response, snapshot *localImageSnapshot) {
+ level, message := imageCheckResultMessageInternal(imageRef, res)
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, level, message, nil, "")
+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, res, snapshot); err != nil {
+ slog.WarnContext(ctx, "Failed to save update result", "imageRef", imageRef, "error", err.Error())
}
- defer s.registryLimiter.Release(registryRecord)
+}
- res, snapshot := s.checkSingleImageInBatchInternal(ctx, resolvedCreds, img.parts, composeBuildRefs)
- if ctx.Err() != nil {
- return ctx.Err()
+// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the
+// activity stream and persists its unchanged result.
+func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress *int) {
+ step := ""
+ if progress != nil {
+ step = "Skipping image update check"
}
- if res != nil {
- res.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — digest pinned, skipped", progress, step)
+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {
+ slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())
}
+}
- s.recordBatchImageCheckInternal(ctx, activityID, img, res, snapshot, recorder)
+// RegisterWorkflows defines the image-check workflow; call it before the host starts.
+func (s *ImageUpdateService) RegisterWorkflows(engine *flow.Engine) error {
+ checkWorkflow, err := engine.Define(flow.Definition{
+ Name: "image-check",
+ Version: 3,
+ Fingerprint: "e508fae9a6d269cc6aaac6375643f761c459e18571aab0efb61d5c0efb94b16d",
+ Concurrency: imageCheckConcurrency,
+ Timeout: timeouts.DefaultImageUpdateScan,
+ Activity: activity.StartActivityRequest{
+ Type: activitytypes.TypeImageUpdateCheck,
+ Queue: true,
+ ResourceType: new("images"),
+ Step: "Checking image updates",
+ LatestMessage: "Image update check started",
+ },
+ Labels: map[string]string{
+ "prepare": "Preparing image update check",
+ "discover": "Discovering images",
+ "check": "Checking images",
+ "tags": "Checking tag policies",
+ "finalize": "Finishing image update check",
+ },
+ Steps: []workflow.StepSpec{
+ workflow.Step("prepare", engine.Handler(s.prepareImageCheckInternal)),
+ workflow.Step("discover", engine.Handler(s.discoverImagesInternal)),
+ workflow.ForEach("check", engine.Handler(s.checkImageInternal),
+ workflow.WithItemsFrom("discover"),
+ workflow.WithInputFrom("prepare"),
+ workflow.WithMaxParallel(imageCheckConcurrency),
+ workflow.WithFailurePolicy(workflow.TolerateFailures),
+ workflow.WithMaxAttempts(imageCheckAttempts),
+ workflow.WithRetryBackoff(imageCheckRetryBackoff, imageCheckRetryBackoffMax),
+ workflow.WithAttemptTimeout(2*time.Minute)),
+ workflow.Step("tags", engine.Handler(s.checkTagPoliciesInternal), workflow.WithInputFrom("discover")),
+ workflow.Step("finalize", engine.Handler(s.finalizeImageCheckInternal), workflow.WithInputFrom("discover", "check")),
+ },
+ // A reported check returns every ref's result, which outgrows the default output limit on large hosts.
+ Options: []workflow.Option{workflow.WithMaxOutputSize(4 << 20), workflow.WithMaxJournalSize(16 << 20)},
+ })
+ if err != nil {
+ return err
+ }
+ s.engine = engine
+ s.checkWorkflow = checkWorkflow
return nil
}
-func (
- s *ImageUpdateService,
-) recordBatchImageCheckInternal(
- ctx context.Context,
- activityID string,
- img batchImage,
- res *imageupdate.Response,
- snapshot *localImageSnapshot,
- recorder *batchImageProgressRecorder,
-) {
- progress := recorder.recordInternal(img.refs, res)
-
- level, message := imageCheckResultMessageInternal(img.canonicalRef, res)
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, level, message, progress, "Checking image")
+// CheckWorkflow is the image-check workflow, for use as a child step.
+func (s *ImageUpdateService) CheckWorkflow() *flow.Workflow { return s.checkWorkflow }
- if err := s.saveUpdateResultWithSnapshotInternal(ctx, img.canonicalRef, res, snapshot); err != nil {
- slog.WarnContext(ctx, "Failed to save update result", "imageRef", img.canonicalRef, "error", err.Error())
- }
+// RunImageCheck checks the requested images and returns the check outcome;
+// failed images are reported as failed targets.
+func (s *ImageUpdateService) RunImageCheck(ctx context.Context, request imageupdate.CheckRequest) (scheduler.Outcome, error) {
+ return s.engine.Run(ctx, s.checkWorkflow, request, activitylib.StartRequest{}, nil)
}
-// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the
-// activity stream and persists its unchanged result.
-func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress int) {
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — digest pinned, skipped", progress, "Skipping image update check")
- if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {
- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())
+// CheckImages checks the requested images and returns each ref's result. Usable credentials replace the stored
+// registries for this check, as they always have.
+func (s *ImageUpdateService) CheckImages(ctx context.Context, request imageupdate.CheckRequest, credentials []containerregistry.Credential) (imageupdate.BatchResponse, error) {
+ if !request.All && len(request.ImageRefs) == 0 {
+ return imageupdate.BatchResponse{}, nil
}
-}
-
-// recordInitialBatchResultsInternal stamps the activity ID on the parse-stage
-// results and persists digest-pinned refs as skipped, since they never reach
-// the registry check stage.
-func (s *ImageUpdateService) recordInitialBatchResultsInternal(ctx context.Context, activityID string, initialResults map[string]*imageupdate.Response) {
- for imageRef, result := range initialResults {
- if result == nil {
+ input := imageCheckInput{CheckRequest: request, Report: true}
+ credentials = slices.DeleteFunc(slices.Clone(credentials), func(credential containerregistry.Credential) bool {
+ return !credential.Enabled || strings.TrimSpace(credential.URL) == "" || strings.TrimSpace(credential.Username) == "" || strings.TrimSpace(credential.Token) == ""
+ })
+ if len(credentials) > 0 {
+ input.CredentialsKey = uuid.NewV7().String()
+ s.requestCredentials.Set(input.CredentialsKey, credentials)
+ }
+ var output imageCheckOutput
+ outcome, err := s.engine.Run(ctx, s.checkWorkflow, input, activitylib.StartRequest{}, &output)
+ if err != nil {
+ // The workflow outlives a caller that stopped waiting, so finalize releases the credentials.
+ return nil, err
+ }
+ s.requestCredentials.Delete(input.CredentialsKey)
+ if outcome.Status == scheduler.Failed || outcome.Status == scheduler.Canceled {
+ return nil, errors.New(cmp.Or(outcome.Message, "image update check failed"))
+ }
+ results := output.Results
+ if results == nil {
+ results = imageupdate.BatchResponse{}
+ }
+ for _, target := range outcome.Targets {
+ if target.ResourceType != "image" {
continue
}
- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- if strings.TrimSpace(result.Error) != "" {
- continue
+ if results[target.ID] == nil {
+ results[target.ID] = &imageupdate.Response{CheckTime: time.Now()}
}
- if _, ok := digest.FromReferenceSuffix(imageRef); !ok {
- continue
- }
-
- s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 5)
+ results[target.ID].Error = target.Message
}
+ // Failed images get their entries first, so their containers' tag results still attach.
+ tags.AttachContainerUpdates(results, output.ContainerUpdates)
+ for _, result := range results {
+ result.ActivityID = mo.EmptyableToOption(outcome.ActivityID).ToPointer()
+ }
+ return results, nil
}
-func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (results map[string]*imageupdate.Response, err error) {
- startBatch := time.Now()
- results = make(map[string]*imageupdate.Response, len(imageRefs))
- if len(imageRefs) == 0 {
- return results, nil
+// MonitoredImageRefs keeps the refs whose containers still permit update checks.
+func (s *ImageUpdateService) MonitoredImageRefs(ctx context.Context, imageRefs []string) ([]string, error) {
+ dockerClient, err := s.dockerClientInternal(ctx)
+ if err != nil {
+ return nil, err
}
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
+ cancel()
+ if err != nil {
+ return nil, fmt.Errorf("list containers for update-check eligibility: %w", err)
+ }
+ eligibility := newMonitoringEligibilityInternal(listed.Items)
+ return slices.DeleteFunc(slices.Clone(imageRefs), func(imageRef string) bool {
+ return !eligibility.imageEligible("", refs.NormalizeImageUpdateRef(imageRef))
+ }), nil
+}
- activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))
- ctx = s.activityService.Track(ctx, activityID)
+// prepareImageCheckInternal loads the compose build refs every check task needs, once per scan.
+func (s *ImageUpdateService) prepareImageCheckInternal(ctx context.Context, _ flow.Task) (any, error) {
+ buildRefs, err := s.composeBuildImageRefsInternal(ctx)
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ return slices.Sorted(maps.Keys(buildRefs)), nil
+}
- // A single deferred finalizer owns the terminal activity write so it runs
- // on success, error, and panic alike — any early return that skipped
- // completion would strand the row in running forever. It reads the Track
- // ctx so a user cancellation still records a cancelled status.
- defer func() {
- if panicErr := utils.PanicToError(recover()); panicErr != nil {
- // Don't re-panic: the caller is the long-lived watcher goroutine.
- err = fmt.Errorf("image update check panicked: %w", panicErr)
- slog.ErrorContext(ctx, "image update check panicked", "activityId", activityID, "error", err)
- }
+func (s *ImageUpdateService) discoverImagesInternal(ctx context.Context, t flow.Task) (any, error) {
+ var request imageCheckInput
+ if err := t.Payload(&request); err != nil {
+ return nil, err
+ }
+ imageRefs := request.ImageRefs
+ if request.All {
+ discovered, err := s.getAllImageRefsInternal(ctx, 0)
if err != nil {
- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())
- return
- }
- successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)
- finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)
- s.completeImageUpdateActivityInternal(ctx, activityID, errorCount == 0, finalMessage)
- }()
-
- if activityID != "" {
- // Bounded slot wait: update-all runs share these slots and can hold
- // them for a long time; parking here forever would wedge every future
- // scan behind the watcher's single-flight gate. On timeout the
- // finalizer above flips the queued row to failed.
- if slotErr := s.activityService.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {
- return results, slotErr
- }
- }
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")
- slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))
-
- regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)
- maps.Copy(results, initialResults)
- s.recordInitialBatchResultsInternal(ctx, activityID, initialResults)
-
- // The aggregate scan gets its own deadline: individual registry RPCs are
- // bounded, but the batch as a whole (including registry limiter waits)
- // was not, and a wedged scan holds its activity slot indefinitely.
- scanCtx, cancelScan := context.WithTimeout(ctx, timeouts.DefaultImageUpdateScan)
- defer cancelScan()
-
- composeBuildRefs, composeErr := s.composeBuildImageRefsInternal(scanCtx)
- if composeErr != nil {
- err = fmt.Errorf("prepare compose build image references: %w", composeErr)
- return results, err
+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("list image references: %w", err))
+ }
+ imageRefs = discovered
}
+ s.appendImageUpdateActivityMessageInternal(ctx, t.ActivityID(), activitytypes.MessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), nil, "")
+ return s.groupImageRefsInternal(imageRefs), nil
+}
- resolvedCreds := s.resolveBatchCredentialsInternal(scanCtx, externalCreds)
-
- slog.DebugContext(ctx, "Resolved batch registry credentials", "credentialCount", len(resolvedCreds), "registryCount", len(regRepos))
-
- recorder := &batchImageProgressRecorder{
- results: results,
- total: len(images),
+// checkImageInternal checks one group of refs. Retryable failures are recorded
+// only on the last attempt so earlier attempts never replace a good result.
+func (s *ImageUpdateService) checkImageInternal(ctx context.Context, t flow.Task) (any, error) {
+ var group []string
+ if err := t.DecodeItem(&group); err != nil || len(group) == 0 {
+ return nil, fmt.Errorf("decode image group: %w", cmp.Or(err, errors.New("empty group")))
}
- g, groupCtx := errgroup.WithContext(scanCtx)
- g.SetLimit(10) // Limit concurrency
-
- for _, img := range images {
- g.Go(func() (checkErr error) {
- // x/sync's errgroup does not recover goroutine panics (they crash
- // the process), so convert them to errors here; the deferred
- // finalizer then records the failed terminal status.
- defer func() {
- if panicErr := utils.PanicToError(recover()); panicErr != nil {
- checkErr = fmt.Errorf("image update check panicked: %w", panicErr)
- slog.ErrorContext(groupCtx, "image update check worker panicked", "activityId", activityID, "imageRef", img.canonicalRef, "error", checkErr)
- }
- }()
- return s.checkBatchImageInternal(groupCtx, activityID, resolvedCreds, img, composeBuildRefs, recorder)
- })
+ var input imageCheckInput
+ if err := t.Payload(&input); err != nil {
+ return nil, err
}
-
- if err = g.Wait(); err != nil {
- if ctx.Err() == nil && errors.Is(scanCtx.Err(), context.DeadlineExceeded) {
- err = fmt.Errorf("image update check timed out after %s: %w", timeouts.DefaultImageUpdateScan, err)
- }
- slog.ErrorContext(ctx, "Batch check error", "error", err)
- return results, err
+ // A manual check answers its caller, who is gone after a restart; its requester was not rechecked here either.
+ if input.Report && t.Recovered() {
+ return nil, errCheckInterrupted
}
+ imageRef := group[0]
+ activityID := t.ActivityID()
+ if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {
+ result.ActivityID = mo.EmptyableToOption(activityID).ToPointer()
+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, nil)
+ return imageCheckResultInternal{Result: kit.Ternary(input.Report, result, nil)}, nil
+ }
+ parts := s.parseImageReference(imageRef)
+ if parts == nil {
+ err := common.Classify(common.ErrValidation, errors.New("invalid image reference format"))
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelError, imageRef+": "+err.Error(), nil, "")
+ return nil, err
+ }
+ if err := s.registryLimiter.Acquire(ctx, parts.Registry); err != nil {
+ return nil, err
+ }
+ defer s.registryLimiter.Release(parts.Registry)
+ var buildRefs []string
+ if err := t.DecodeOutput("prepare", &buildRefs); err != nil {
+ return nil, err
+ }
+ composeBuildRefs := make(map[string]struct{}, len(buildRefs))
+ for _, buildRef := range buildRefs {
+ composeBuildRefs[buildRef] = struct{}{}
+ }
+ res, snapshot, checkErr := s.checkSingleImageInBatchInternal(ctx, s.resolveBatchCredentialsInternal(ctx, t, input), parts, composeBuildRefs)
+ // Shutdown hands the task back for redelivery rather than recording it as a failed check.
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ // A reported check answers a waiting caller, so it gets one attempt as the batch endpoint always has.
+ if checkErr != nil && flow.Retryable(checkErr) && !input.Report && t.Attempt() < imageCheckAttempts {
+ return nil, checkErr
+ }
+ res.ActivityID = mo.EmptyableToOption(activityID).ToPointer()
+ s.recordImageCheckInternal(ctx, activityID, imageRef, res, snapshot)
+ if checkErr != nil && input.Report {
+ // The waiting caller gets the failed response, auth details included, alongside its error.
+ return imageCheckResultInternal{Result: res, Failure: checkErr.Error()}, nil
+ }
+ if checkErr != nil {
+ return nil, checkErr
+ }
+ return imageCheckResultInternal{Local: res.UpdateType == UpdateTypeLocal, Result: kit.Ternary(input.Report, res, nil)}, nil
+}
+// checkTagPoliciesInternal checks container tag policies for the non-local refs.
+func (s *ImageUpdateService) checkTagPoliciesInternal(ctx context.Context, t flow.Task) (any, error) {
+ groups, results, err := imageCheckResultsInternal(t)
+ if err != nil {
+ return nil, err
+ }
var tagRefs []string
- for _, imageRef := range imageRefs {
- if res := results[imageRef]; res != nil && res.UpdateType != UpdateTypeLocal {
- tagRefs = append(tagRefs, imageRef)
+ for index, group := range groups {
+ if !results[index].Value.Local {
+ tagRefs = append(tagRefs, group...)
}
}
- containerUpdates, tagErr := s.checkContainerTagUpdatesInternal(scanCtx, tagRefs, resolvedCreds)
- if tagErr != nil {
- return results, tagErr
+ var input imageCheckInput
+ if payloadErr := t.Payload(&input); payloadErr != nil {
+ return nil, payloadErr
+ }
+ if input.Report && t.Recovered() {
+ return tagCheckOutput{Failures: []scheduler.TargetOutcome{}}, nil
+ }
+ containerUpdates, err := s.checkContainerTagUpdatesInternal(ctx, tagRefs, s.resolveBatchCredentialsInternal(ctx, t, input))
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ output := tagCheckOutput{Failures: []scheduler.TargetOutcome{}}
+ for containerID, update := range containerUpdates {
+ if update.Error != "" {
+ output.Failures = append(output.Failures, scheduler.TargetOutcome{ResourceType: "container", ID: containerID, Status: scheduler.Failed, Message: update.Error})
+ }
}
- tags.AttachContainerUpdates(results, containerUpdates)
- successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)
- slog.InfoContext(ctx, "Batch image update check completed",
- "totalImages", len(imageRefs),
- "successCount", successCount,
- "errorCount", errorCount,
- "duration", time.Since(startBatch))
+ if input.Report {
+ output.Updates = containerUpdates
+ }
+ return output, nil
+}
+func (s *ImageUpdateService) finalizeImageCheckInternal(ctx context.Context, t flow.Task) (any, error) {
+ var request imageCheckInput
+ if err := t.Payload(&request); err != nil {
+ return nil, err
+ }
+ // Every check task has run, so the caller's credentials are no longer needed.
+ s.requestCredentials.Delete(request.CredentialsKey)
+ groups, results, err := imageCheckResultsInternal(t)
+ if err != nil {
+ return nil, err
+ }
+ var tagged tagCheckOutput
+ if decodeErr := t.DecodeOutput("tags", &tagged); decodeErr != nil {
+ return nil, decodeErr
+ }
+ tagFailures := tagged.Failures
+ checked, failed := 0, 0
+ var targets []scheduler.TargetOutcome
+ for index, group := range groups {
+ checked += len(group)
+ if results[index].Err == "" {
+ continue
+ }
+ failed += len(group)
+ for _, imageRef := range group {
+ targets = append(targets, scheduler.TargetOutcome{ResourceType: "image", ID: imageRef, Status: scheduler.Failed, Message: results[index].Err})
+ }
+ }
+ targets = append(targets, tagFailures...)
// A failed flush is already logged; the records stay pending for the next one.
_ = s.SendBatchUpdateNotifications(ctx)
+ if request.All && (!request.Report || !t.Recovered()) {
+ if cleanupErr := s.CleanupOrphanedRecords(ctx); cleanupErr != nil {
+ slog.WarnContext(ctx, "failed to cleanup orphaned image update records after check-all", "error", cleanupErr.Error())
+ }
+ }
+ outcome := scheduler.Outcome{
+ Status: scheduler.Succeeded,
+ Message: fmt.Sprintf("Image update check completed: %d checked, %d errors", checked, failed),
+ Targets: targets,
+ }
+ if len(tagFailures) > 0 {
+ outcome.Message += fmt.Sprintf(", %d tag policy errors", len(tagFailures))
+ }
+ if len(targets) > 0 {
+ outcome.Status = scheduler.Partial
+ }
+ slog.InfoContext(ctx, "Image update check completed", "totalImages", checked, "failedImages", failed, "failedContainers", len(tagFailures))
+ if !request.Report {
+ return outcome, nil
+ }
+ output := imageCheckOutput{Outcome: outcome, Results: imageupdate.BatchResponse{}, ContainerUpdates: tagged.Updates}
+ for index, group := range groups {
+ if result := results[index].Value.Result; result != nil {
+ for _, imageRef := range group {
+ output.Results[imageRef] = new(*result)
+ }
+ }
+ }
+ return output, nil
+}
- return results, nil
+// imageCheckResultsInternal reads the discovered groups and their check slots.
+func imageCheckResultsInternal(t flow.Task) ([][]string, []flow.Result[imageCheckResultInternal], error) {
+ var groups [][]string
+ if err := t.DecodeOutput("discover", &groups); err != nil {
+ return nil, nil, err
+ }
+ results, err := flow.Results[imageCheckResultInternal](t, "check")
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(results) != len(groups) {
+ return nil, nil, fmt.Errorf("image check produced %d results for %d groups", len(results), len(groups))
+ }
+ for index := range results {
+ results[index].Err = cmp.Or(results[index].Err, results[index].Value.Failure)
+ }
+ return groups, results, nil
}
// SendBatchUpdateNotifications delivers pending update notifications. It
@@ -1837,21 +1973,7 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) e
for imageID, record := range unnotifiedUpdates {
imageRef := containerUpdateNotificationKeyInternal(record)
- updatesToNotify[imageRef] = &imageupdate.Response{
- HasUpdate: record.HasUpdate,
- UpdateType: record.UpdateType,
- CurrentVersion: record.CurrentVersion,
- LatestVersion: mo.PointerToOption(record.LatestVersion).OrEmpty(),
- CurrentDigest: mo.PointerToOption(record.CurrentDigest).OrEmpty(),
- LatestDigest: mo.PointerToOption(record.LatestDigest).OrEmpty(),
- CheckTime: record.CheckTime,
- ResponseTimeMs: record.ResponseTimeMs,
- Error: mo.PointerToOption(record.LastError).OrEmpty(),
- AuthMethod: mo.PointerToOption(record.AuthMethod).OrEmpty(),
- AuthUsername: mo.PointerToOption(record.AuthUsername).OrEmpty(),
- AuthRegistry: mo.PointerToOption(record.AuthRegistry).OrEmpty(),
- UsedCredential: record.UsedCredential,
- }
+ updatesToNotify[imageRef] = responseFromRecordInternal(record)
imageIDsToMark = append(imageIDsToMark, imageID)
}
@@ -1877,28 +1999,6 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) e
return nil
}
-func (s *ImageUpdateService) CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error) {
- imageRefs, err := s.getAllImageRefsInternal(ctx, limit)
- if err != nil {
- return nil, fmt.Errorf("failed to get image references: %w", err)
- }
-
- if len(imageRefs) == 0 {
- return make(map[string]*imageupdate.Response), nil
- }
-
- results, err := s.CheckMultipleImages(ctx, imageRefs, externalCreds)
- if err != nil {
- return nil, err
- }
-
- if cleanupOrphanedRecordsErr := s.CleanupOrphanedRecords(ctx); cleanupOrphanedRecordsErr != nil {
- slog.WarnContext(ctx, "failed to cleanup orphaned image update records after check-all", "error", cleanupOrphanedRecordsErr.Error())
- }
-
- return results, nil
-}
-
// DeleteRecordsForImages removes image-level rows (keyed by id) and
// container/project rows (keyed by image_id) for the given images.
func (s *ImageUpdateService) DeleteRecordsForImages(ctx context.Context, imageIDs []string) error {
@@ -2077,6 +2177,24 @@ func (s *ImageUpdateService) saveContainerTagResultInternal(ctx context.Context,
})
}
+func responseFromRecordInternal(record *ImageUpdateRecord) *imageupdate.Response {
+ return &imageupdate.Response{
+ HasUpdate: record.HasUpdate,
+ UpdateType: record.UpdateType,
+ CurrentVersion: record.CurrentVersion,
+ LatestVersion: mo.PointerToOption(record.LatestVersion).OrEmpty(),
+ CurrentDigest: mo.PointerToOption(record.CurrentDigest).OrEmpty(),
+ LatestDigest: mo.PointerToOption(record.LatestDigest).OrEmpty(),
+ CheckTime: record.CheckTime,
+ ResponseTimeMs: record.ResponseTimeMs,
+ Error: mo.PointerToOption(record.LastError).OrEmpty(),
+ AuthMethod: mo.PointerToOption(record.AuthMethod).OrEmpty(),
+ AuthUsername: mo.PointerToOption(record.AuthUsername).OrEmpty(),
+ AuthRegistry: mo.PointerToOption(record.AuthRegistry).OrEmpty(),
+ UsedCredential: record.UsedCredential,
+ }
+}
+
func containerUpdateNotificationKeyInternal(record *ImageUpdateRecord) string {
imageRef := fmt.Sprintf("%s:%s", record.Repository, record.Tag)
if record.ContainerID != "" {
diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.go
index 4cafb6bef..cc3b13d9c 100644
--- a/backend/internal/imageupdate/service_test.go
+++ b/backend/internal/imageupdate/service_test.go
@@ -15,8 +15,8 @@ import (
"github.com/distribution/reference"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
- "github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/libtnb/sqlite"
"github.com/moby/moby/api/pkg/authconfig"
"github.com/moby/moby/api/types/container"
@@ -41,6 +41,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
@@ -406,16 +409,10 @@ func TestImageUpdateService_CheckImageUpdate_ComposeBuildSkipsRegistryWithRepoDi
assert.Nil(t, saved.LastError)
}
-func TestImageUpdateService_CheckMultipleImages_ComposeBuildSkipsRegistryWithRepoDigests(t *testing.T) {
+func TestImageUpdateService_CheckImages_ComposeBuildSkipsRegistryWithRepoDigests(t *testing.T) {
svc, registryCalls := newComposeBuildImageUpdateServiceInternal(t)
- credentials := []containerregistry.Credential{{
- URL: "https://index.docker.io/v1/",
- Username: "unused",
- Token: "unused",
- Enabled: true,
- }}
- results, err := svc.CheckMultipleImages(t.Context(), []string{"test2:latest"}, credentials)
+ results, err := checkImagesInternal(t, t.Context(), svc, "test2:latest")
require.NoError(t, err)
result := results["test2:latest"]
require.NotNil(t, result)
@@ -425,7 +422,7 @@ func TestImageUpdateService_CheckMultipleImages_ComposeBuildSkipsRegistryWithRep
assert.Zero(t, registryCalls.Load())
}
-func TestImageUpdateService_CheckMultipleImages_ComposeBuildMissingLocallySkipsRegistry(t *testing.T) {
+func TestImageUpdateService_CheckImages_ComposeBuildMissingLocallySkipsRegistry(t *testing.T) {
db := setupImageUpdateTestDB(t)
buildRefsJSON := `["test2:latest"]`
require.NoError(t, db.Create(&testProjectRow{
@@ -461,7 +458,7 @@ func TestImageUpdateService_CheckMultipleImages_ComposeBuildMissingLocallySkipsR
eventService := event.NewEventService(db, nil, nil)
svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{"test2:latest"}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, "test2:latest")
require.NoError(t, err)
result := results["test2:latest"]
require.NotNil(t, result)
@@ -540,10 +537,10 @@ func TestImageUpdateService_CheckImageUpdate_ArcaneLocalHostSkipsRegistry(t *tes
assert.Nil(t, saved.LastError)
}
-func TestImageUpdateService_CheckMultipleImages_ArcaneLocalMissingImageSkipsRegistry(t *testing.T) {
+func TestImageUpdateService_CheckImages_ArcaneLocalMissingImageSkipsRegistry(t *testing.T) {
svc, registryCalls := newArcaneLocalImageUpdateServiceInternal(t, false)
- results, err := svc.CheckMultipleImages(t.Context(), []string{"arcane.local/demo-2ab41b29/worker:latest"}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, "arcane.local/demo-2ab41b29/worker:latest")
require.NoError(t, err)
result := results["arcane.local/demo-2ab41b29/worker:latest"]
require.NotNil(t, result)
@@ -553,10 +550,10 @@ func TestImageUpdateService_CheckMultipleImages_ArcaneLocalMissingImageSkipsRegi
assert.Zero(t, registryCalls.Load())
}
-func TestImageUpdateService_CheckMultipleImages_OtherDottedRegistryStillChecked(t *testing.T) {
+func TestImageUpdateService_CheckImages_OtherDottedRegistryStillChecked(t *testing.T) {
svc, registryCalls := newArcaneLocalImageUpdateServiceInternal(t, false)
- results, err := svc.CheckMultipleImages(t.Context(), []string{"registry.local/team/app:latest"}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, "registry.local/team/app:latest")
require.NoError(t, err)
result := results["registry.local/team/app:latest"]
require.NotNil(t, result)
@@ -783,20 +780,15 @@ func TestImageUpdateService_CheckImageUpdate_SkipsDigestPinnedReferenceInternal(
assert.Empty(t, result.Error)
}
-func TestImageUpdateService_CheckMultipleImages_SkipsDigestPinnedReferenceInternal(t *testing.T) {
+func TestImageUpdateService_CheckImages_SkipsDigestPinnedReferenceInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
pinnedDigest := digest.FromString("batch-pinned-newt").String()
imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest
svc := NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)
- regRepos, initialResults, grouped := svc.parseAndGroupImagesInternal([]string{imageRef})
- require.Empty(t, regRepos)
- require.Empty(t, grouped)
- require.Contains(t, initialResults, imageRef)
- require.NotNil(t, initialResults[imageRef])
- assert.Empty(t, initialResults[imageRef].Error)
+ require.Equal(t, [][]string{{imageRef}}, svc.groupImageRefsInternal([]string{imageRef}))
- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)
require.NoError(t, err)
require.Contains(t, results, imageRef)
require.NotNil(t, results[imageRef])
@@ -807,7 +799,7 @@ func TestImageUpdateService_CheckMultipleImages_SkipsDigestPinnedReferenceIntern
assert.Empty(t, results[imageRef].Error)
}
-func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceClearsStaleErrorInternal(t *testing.T) {
+func TestImageUpdateService_CheckImages_SkippedDigestPinnedReferenceClearsStaleErrorInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
pinnedDigest := digest.FromString("stale-pinned-newt").String()
imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest
@@ -825,7 +817,7 @@ func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceClea
}).Error)
svc := NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)
require.NoError(t, err)
require.Contains(t, results, imageRef)
assert.Empty(t, results[imageRef].Error)
@@ -838,7 +830,7 @@ func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceClea
assert.Equal(t, pinnedDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())
}
-func TestImageUpdateService_CheckMultipleImages_DigestPinnedTagPreservedWhenLocalImageHasNoRepoTags(t *testing.T) {
+func TestImageUpdateService_CheckImages_DigestPinnedTagPreservedWhenLocalImageHasNoRepoTags(t *testing.T) {
db := setupImageUpdateTestDB(t)
pinnedDigest := digest.FromString("valkey-pinned").String()
@@ -853,7 +845,7 @@ func TestImageUpdateService_CheckMultipleImages_DigestPinnedTagPreservedWhenLoca
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
svc := NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)
require.NoError(t, err)
require.Contains(t, results, imageRef)
assert.False(t, results[imageRef].HasUpdate)
@@ -930,7 +922,7 @@ func TestImageUpdateService_CheckImageUpdate_UsesRegistryFallback(t *testing.T)
assert.Equal(t, remoteDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())
}
-func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.T) {
+func TestImageUpdateService_CheckImages_UsesRegistryFallback(t *testing.T) {
db := setupImageUpdateTestDB(t)
localDigest := digest.FromString("batchlocal").String()
remoteDigest := digest.FromString("batchremote").String()
@@ -955,7 +947,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.
eventService := event.NewEventService(db, nil, nil)
svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)
require.NoError(t, err)
require.Contains(t, results, imageRef)
@@ -972,7 +964,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.
assert.Equal(t, remoteDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())
}
-func TestImageUpdateService_CheckMultipleImagesCompletesActivityWhenRequestContextCanceledInternal(t *testing.T) {
+func TestImageUpdateService_CheckImagesCompletesActivityAfterRequestContextCanceledInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
@@ -982,44 +974,43 @@ func TestImageUpdateService_CheckMultipleImagesCompletesActivityWhenRequestConte
for range 5 {
require.NoError(t, svc.registryLimiter.Acquire(t.Context(), "docker.io"))
}
- defer func() {
- for range 5 {
- svc.registryLimiter.Release("docker.io")
- }
- }()
+ startCheckWorkflowInternal(t, svc)
ctx, cancel := context.WithCancel(t.Context())
errCh := make(chan error, 1)
go func() {
- _, err := svc.CheckMultipleImages(ctx, []string{"nginx:latest"}, nil)
+ _, err := svc.CheckImages(ctx, imageupdate.CheckRequest{ImageRefs: []string{"nginx:latest"}}, nil)
errCh <- err
}()
var localActivity activity.Activity
require.Eventually(t, func() bool {
return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil
- }, time.Second, 10*time.Millisecond)
+ }, 5*time.Second, 10*time.Millisecond)
cancel()
select {
case err := <-errCh:
require.ErrorIs(t, err, context.Canceled)
- case <-time.After(time.Second):
+ case <-time.After(5 * time.Second):
require.FailNow(t, "timed out waiting for image update check to return")
}
+ // The durable check outlives the request and still finalizes its activity.
+ for range 5 {
+ svc.registryLimiter.Release("docker.io")
+ }
require.Eventually(t, func() bool {
if err := db.First(&localActivity, "id = ?", localActivity.ID).Error; err != nil {
return false
}
return localActivity.Status == activitytypes.StatusFailed
- }, time.Second, 10*time.Millisecond)
- assert.Equal(t, "Image update check complete", localActivity.Step)
- assert.Contains(t, localActivity.LatestMessage, "Image update check failed")
+ }, 15*time.Second, 10*time.Millisecond)
+ assert.NotNil(t, localActivity.EndedAt)
}
-func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInternal(t *testing.T) {
+func TestImageUpdateService_CheckImagesRetriesStalledRegistryCheckInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
@@ -1036,7 +1027,8 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter
}, nil
}, nil, nil)
- parentCtx, cancel := context.WithTimeout(t.Context(), 2500*time.Millisecond)
+ shortenImageCheckRetryBackoffInternal(t)
+ parentCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
defer cancel()
svc := NewImageUpdateService(
@@ -1053,25 +1045,26 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter
)
start := time.Now()
- results, err := svc.CheckMultipleImages(parentCtx, []string{"registry.example.com/team/app:1.2.3"}, nil)
+ results, err := checkImagesInternal(t, parentCtx, svc, "registry.example.com/team/app:1.2.3")
elapsed := time.Since(start)
require.NoError(t, err)
- require.Less(t, elapsed, 2*time.Second)
+ // Each of the three attempts is bounded by the one-second registry timeout.
+ require.Less(t, elapsed, 8*time.Second)
require.Contains(t, results, "registry.example.com/team/app:1.2.3")
require.NotNil(t, results["registry.example.com/team/app:1.2.3"])
require.Contains(t, results["registry.example.com/team/app:1.2.3"].Error, context.DeadlineExceeded.Error())
var localActivity activity.Activity
- require.NoError(t, db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error)
+ require.Eventually(t, func() bool {
+ return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil && localActivity.EndedAt != nil
+ }, 5*time.Second, 10*time.Millisecond)
require.Equal(t, activitytypes.StatusFailed, localActivity.Status)
- require.NotNil(t, localActivity.EndedAt)
require.NotNil(t, localActivity.DurationMs)
- require.Equal(t, "Image update check complete", localActivity.Step)
- require.Contains(t, localActivity.LatestMessage, "0 checked, 1 errors")
+ require.Contains(t, localActivity.LatestMessage, "1 checked, 1 errors")
}
-func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(t *testing.T) {
+func TestImageUpdateService_CheckImagesPanicFailsOnlyThatImageInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
@@ -1100,16 +1093,16 @@ func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(
activityService,
)
- _, err := svc.CheckMultipleImages(t.Context(), []string{"registry.example.com/team/app:1.2.3"}, nil)
- require.Error(t, err)
- require.Contains(t, err.Error(), "image update check panicked")
- require.Contains(t, err.Error(), "registry check exploded")
+ results, err := checkImagesInternal(t, t.Context(), svc, "registry.example.com/team/app:1.2.3")
+ require.NoError(t, err)
+ require.Contains(t, results["registry.example.com/team/app:1.2.3"].Error, "task panicked: registry check exploded")
var localActivity activity.Activity
- require.NoError(t, db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error)
+ require.Eventually(t, func() bool {
+ return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil && localActivity.EndedAt != nil
+ }, 5*time.Second, 10*time.Millisecond)
require.Equal(t, activitytypes.StatusFailed, localActivity.Status)
- require.NotNil(t, localActivity.EndedAt)
- require.Contains(t, localActivity.LatestMessage, "Image update check failed")
+ require.Contains(t, localActivity.LatestMessage, "1 errors")
}
func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeoutInternal(t *testing.T) {
@@ -1148,7 +1141,7 @@ func TestImageUpdateService_InspectLocalImageSnapshotUsesDockerAPITimeoutInterna
require.Contains(t, err.Error(), context.DeadlineExceeded.Error())
}
-func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstAttempt(t *testing.T) {
+func TestImageUpdateService_CheckImages_UsesDockerHubCredentialsOnFirstAttempt(t *testing.T) {
db := setupImageUpdateRegistryTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&ImageUpdateRecord{}, &event.Event{}, &testProjectRow{}))
createImageUpdateTestPullRegistryInternal(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")
@@ -1184,7 +1177,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstA
eventService := event.NewEventService(db, nil, nil)
svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{"docker.io/library/registry:3"}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, "docker.io/library/registry:3")
require.NoError(t, err)
require.Contains(t, results, "docker.io/library/registry:3")
@@ -1200,7 +1193,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstA
assert.True(t, result.UsedCredential)
}
-func TestImageUpdateService_CheckMultipleImages_ReportsNotPulledWhenLocalImageMissing(t *testing.T) {
+func TestImageUpdateService_CheckImages_ReportsNotPulledWhenLocalImageMissing(t *testing.T) {
db := setupImageUpdateTestDB(t)
remoteDigest := digest.FromString("registry-only-remote").String()
@@ -1223,7 +1216,7 @@ func TestImageUpdateService_CheckMultipleImages_ReportsNotPulledWhenLocalImageMi
eventService := event.NewEventService(db, nil, nil)
svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)
- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)
+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)
require.NoError(t, err)
require.Contains(t, results, imageRef)
require.NotNil(t, results[imageRef])
@@ -2122,26 +2115,10 @@ func TestImageUpdateService_ParseAndGroupImages_DedupesNormalizedRefs(t *testing
"docker.io/library/redis:7",
}
- regRepos, initialResults, grouped := svc.parseAndGroupImagesInternal(refs)
- require.Empty(t, initialResults)
+ grouped := svc.groupImageRefsInternal(refs)
require.Len(t, grouped, 2)
- require.Contains(t, regRepos, "docker.io")
- require.Len(t, regRepos["docker.io"], 2)
-
- firstRefSet := map[string]struct{}{}
- for _, imageRef := range grouped[0].refs {
- firstRefSet[imageRef] = struct{}{}
- }
- secondRefSet := map[string]struct{}{}
- for _, imageRef := range grouped[1].refs {
- secondRefSet[imageRef] = struct{}{}
- }
-
- // Each normalized image should only be checked once, while retaining all aliases.
- assert.True(t, (containsAll(firstRefSet, "nginx:latest", "docker.io/library/nginx:latest") &&
- containsAll(secondRefSet, "redis:7", "docker.io/library/redis:7")) ||
- (containsAll(secondRefSet, "nginx:latest", "docker.io/library/nginx:latest") &&
- containsAll(firstRefSet, "redis:7", "docker.io/library/redis:7")))
+ assert.ElementsMatch(t, []string{"nginx:latest", "docker.io/library/nginx:latest"}, grouped[0])
+ assert.ElementsMatch(t, []string{"redis:7", "docker.io/library/redis:7"}, grouped[1])
}
func newImageUpdateTestSettingsServiceInternal(t *testing.T, registryTimeout, dockerAPITimeout string) *settings.SettingsService {
@@ -2183,15 +2160,6 @@ func newBlockedDockerAPIServerInternal(t *testing.T, pathContains string) *httpt
return server
}
-func containsAll(set map[string]struct{}, refs ...string) bool {
- for _, imageRef := range refs {
- if _, ok := set[imageRef]; !ok {
- return false
- }
- }
- return true
-}
-
func newImageUpdateDiscoveryServerInternal(
t *testing.T,
images []image.Summary,
@@ -3017,3 +2985,163 @@ func TestContainerTagChecksPersistResultsFinishedBeforeScanDeadlineInternal(t *t
require.Equal(t, "1.1.0", mo.PointerToOption(records[0].LatestVersion).OrEmpty())
require.Empty(t, mo.PointerToOption(records[0].LastError).OrEmpty(), "a stored result must not be overwritten by the cancellation")
}
+
+func TestImageUpdateService_RunImageCheckExhaustedRateLimitKeepsPreviousResult(t *testing.T) {
+ // Serial: it shortens the package-wide retry backoff.
+ const imageRef = "registry.example.com/team/app:1"
+ var calls atomic.Int32
+ svc, db := newWorkflowCheckServiceInternal(t, map[string]string{imageRef: "sha256:app"}, func(string) (client.DistributionInspectResult, error) {
+ calls.Add(1)
+ return client.DistributionInspectResult{}, errors.New("toomanyrequests: rate limit exceeded")
+ })
+ shortenImageCheckRetryBackoffInternal(t)
+ previousDigest := digest.FromString("previous-remote").String()
+ require.NoError(t, db.Create(&ImageUpdateRecord{
+ ID: "sha256:app", Repository: "registry.example.com/team/app", Tag: "1", HasUpdate: true,
+ UpdateType: UpdateTypeDigest, CurrentVersion: "1", LatestDigest: &previousDigest, CheckTime: time.Now().Add(-time.Hour),
+ }).Error)
+
+ // A scheduled check retries across workflow attempts; a reported one answers its caller after a single attempt.
+ if svc.engine == nil {
+ startCheckWorkflowInternal(t, svc)
+ }
+ outcome, err := svc.RunImageCheck(t.Context(), imageupdate.CheckRequest{ImageRefs: []string{imageRef}})
+ require.NoError(t, err)
+ require.Len(t, outcome.Targets, 1)
+ require.Contains(t, outcome.Targets[0].Message, "toomanyrequests")
+ require.Equal(t, int32(5*imageCheckAttempts), calls.Load(), "each workflow attempt spends the registry's own retry budget")
+
+ var saved ImageUpdateRecord
+ require.NoError(t, db.First(&saved, "id = ?", "sha256:app").Error)
+ assert.True(t, saved.HasUpdate)
+ assert.Nil(t, saved.LastError)
+ assert.Equal(t, previousDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())
+}
+
+func TestImageUpdateService_RunImageCheckReportsPartialFailure(t *testing.T) {
+ const okRef = "registry.example.com/team/app:1"
+ const deniedRef = "registry.example.com/team/private:1"
+ remoteDigest := digest.FromString("remote").String()
+ svc, _ := newWorkflowCheckServiceInternal(t, map[string]string{okRef: "sha256:app"}, func(imageRef string) (client.DistributionInspectResult, error) {
+ if strings.Contains(imageRef, "private") {
+ return client.DistributionInspectResult{}, errors.New("unauthorized: authentication required")
+ }
+ return client.DistributionInspectResult{Descriptor: v1.Descriptor{Digest: digest.Digest(remoteDigest)}}, nil
+ })
+ harness := startCheckWorkflowInternal(t, svc)
+ flowtest.AssertDefinitions(t, harness)
+
+ request := imageupdate.CheckRequest{ImageRefs: []string{okRef, deniedRef}}
+ outcome, err := svc.RunImageCheck(t.Context(), request)
+ require.NoError(t, err)
+ require.Equal(t, scheduler.Partial, outcome.Status)
+ require.Len(t, outcome.Targets, 1)
+ assert.Equal(t, deniedRef, outcome.Targets[0].ID)
+ assert.Equal(t, scheduler.Failed, outcome.Targets[0].Status)
+ require.Len(t, outcome.Steps, 5)
+
+ results, err := svc.CheckImages(t.Context(), request, nil)
+ require.NoError(t, err)
+ assert.True(t, results[okRef].HasUpdate)
+ assert.Empty(t, results[okRef].Error)
+ assert.Equal(t, remoteDigest, results[okRef].LatestDigest)
+ assert.Contains(t, results[deniedRef].Error, "unauthorized")
+}
+
+// startCheckWorkflowInternal registers the image-check workflow on an isolated host.
+func startCheckWorkflowInternal(t *testing.T, svc *ImageUpdateService) *flowtest.Harness {
+ t.Helper()
+ var activities flow.Activities = noopActivitiesInternal{}
+ if svc.activityService != nil {
+ activities = svc.activityService
+ }
+ harness := flowtest.New(t, activities)
+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ return harness
+}
+
+// checkImagesInternal runs refs through the image-check workflow.
+func checkImagesInternal(t *testing.T, ctx context.Context, svc *ImageUpdateService, imageRefs ...string) (imageupdate.BatchResponse, error) {
+ t.Helper()
+ if svc.engine == nil {
+ startCheckWorkflowInternal(t, svc)
+ }
+ return svc.CheckImages(ctx, imageupdate.CheckRequest{ImageRefs: imageRefs}, nil)
+}
+
+// newWorkflowCheckServiceInternal serves one local image per repository named in
+// localImages and answers registry lookups through inspect.
+func newWorkflowCheckServiceInternal(
+ t *testing.T,
+ localImages map[string]string,
+ inspect func(imageRef string) (client.DistributionInspectResult, error),
+) (*ImageUpdateService, *database.DB) {
+ t.Helper()
+ db := setupImageUpdateTestDB(t)
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if strings.HasSuffix(r.URL.Path, "/containers/json") {
+ assert.NoError(t, json.NewEncoder(w).Encode([]container.Summary{}))
+ return
+ }
+ for imageRef, imageID := range localImages {
+ if strings.Contains(r.URL.Path, "/images/"+imageRef+"/json") {
+ repository := imageRef[:strings.LastIndex(imageRef, ":")]
+ assert.NoError(t, json.NewEncoder(w).Encode(image.InspectResponse{
+ ID: imageID,
+ RepoTags: []string{imageRef},
+ RepoDigests: []string{repository + "@" + digest.FromString(imageID).String()},
+ }))
+ return
+ }
+ }
+ http.Error(w, "No such image", http.StatusNotFound)
+ }))
+ t.Cleanup(server.Close)
+ registryService := registry.NewContainerRegistryService(db, func(context.Context) (registry.RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(_ context.Context, imageRef string, _ client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return inspect(imageRef)
+ },
+ }, nil
+ }, nil, nil)
+ dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
+ return NewImageUpdateService(db, nil, registryService, dockerService, event.NewEventService(db, nil, nil), nil, nil), db
+}
+
+func shortenImageCheckRetryBackoffInternal(t *testing.T) {
+ t.Helper()
+ initial, maximum := imageCheckRetryBackoff, imageCheckRetryBackoffMax
+ imageCheckRetryBackoff, imageCheckRetryBackoffMax = 10*time.Millisecond, 20*time.Millisecond
+ t.Cleanup(func() { imageCheckRetryBackoff, imageCheckRetryBackoffMax = initial, maximum })
+}
+
+// noopActivitiesInternal stands in for the activity service in tests that do not inspect activities.
+type noopActivitiesInternal struct{}
+
+func (noopActivitiesInternal) StartActivity(_ context.Context, req activitylib.StartRequest) (*activitytypes.Activity, error) {
+ return &activitytypes.Activity{ID: req.ID}, nil
+}
+
+func (noopActivitiesInternal) CompleteActivity(_ context.Context, activityID string, status activitytypes.Status, _ string, _ *string, _ ...string) (*activitytypes.Activity, error) {
+ return &activitytypes.Activity{ID: activityID, Status: status}, nil
+}
+
+func (noopActivitiesInternal) AppendMessage(context.Context, string, activitylib.AppendMessageRequest) (*activitytypes.Message, error) {
+ return &activitytypes.Message{}, nil
+}
+
+func (noopActivitiesInternal) AppendMessages(context.Context, string, []activitylib.AppendMessageRequest) ([]activitytypes.Message, error) {
+ return nil, nil
+}
+
+func (noopActivitiesInternal) Track(ctx context.Context, _ string) context.Context { return ctx }
+
+func (noopActivitiesInternal) UpdateActivity(_ context.Context, activityID string, _ activitylib.UpdateRequest) (*activitytypes.Activity, error) {
+ return &activitytypes.Activity{ID: activityID}, nil
+}
+
+func (noopActivitiesInternal) AwaitActivitySlotBounded(context.Context, string, string) error {
+ return nil
+}
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index 376f476b3..c3620bde7 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -33,10 +33,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
// JobService manages configuration for background job schedules.
@@ -624,7 +622,7 @@ func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run)
return scheduler.Outcome{Status: scheduler.Waiting, Message: "Waiting for Docker"}, err
}
}
- ctx = s.runContextInternal(ctx, run)
+ ctx = s.runs.ExecutionContext(ctx, run, "")
if run.JobID == "environment-health" && s.RunEnvironmentHealthNow != nil {
err := s.RunEnvironmentHealthNow(ctx)
return classifyOutcomeInternal(run.JobID, scheduler.Outcome{}, err)
@@ -697,7 +695,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Run
if s.scheduler != nil {
if job, ok := s.scheduler.GetJob(run.JobID); ok {
if reconciler, localOk := job.(scheduler.Reconciler); localOk {
- return reconciler.Reconcile(s.runContextInternal(ctx, run), run)
+ return reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)
}
}
}
@@ -723,7 +721,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Run
}
if job, ok := s.scheduler.GetJob(run.JobID); ok {
if reconciler, localOk2 := job.(scheduler.Reconciler); localOk2 {
- outcome, err := reconciler.Reconcile(s.runContextInternal(ctx, run), run)
+ outcome, err := reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)
if err != nil {
slog.WarnContext(ctx, "Job reconciliation requires attention", "runId", run.ID, "error", err)
}
@@ -733,29 +731,6 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Run
return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil
}
-func (s *JobService) runContextInternal(ctx context.Context, run scheduler.Run) context.Context {
- ctx = utils.WithActivityBatchID(ctx, run.ID)
- ctx = jobcontext.WithExecution(ctx, run, func(target scheduler.TargetOutcome) error {
- return s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {
- if current.Status != scheduler.Running || current.Owner != run.Owner {
- return runs.ErrRunConflict
- }
- for index := range current.Outcome.Targets {
- if current.Outcome.Targets[index].ID == target.ID {
- if len(target.RecoveryData) == 0 {
- target.RecoveryData = current.Outcome.Targets[index].RecoveryData
- }
- current.Outcome.Targets[index] = target
- return nil
- }
- }
- current.Outcome.Targets = append(current.Outcome.Targets, target)
- return nil
- })
- })
- return ctx
-}
-
func requiresDockerInternal(jobID string) bool {
switch jobID {
case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":
diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.go
index f24d9163b..3f042d277 100644
--- a/backend/internal/job/service_test.go
+++ b/backend/internal/job/service_test.go
@@ -642,7 +642,7 @@ func TestJobActivityVisibilityAndGrouping(t *testing.T) {
require.NoError(t, getActivityDetailErr)
require.Equal(t, test.environment, detail.Activity.Metadata["environmentId"])
}
- child, err := activities.StartActivity(svc.runContextInternal(t.Context(), run), activity.StartActivityRequest{Type: activitytypes.TypeImagePull})
+ child, err := activities.StartActivity(svc.runs.ExecutionContext(t.Context(), run, ""), activity.StartActivityRequest{Type: activitytypes.TypeImagePull})
require.NoError(t, err)
require.Equal(t, run.ID, *child.BatchID)
})
diff --git a/backend/internal/network/handler.go b/backend/internal/network/handler.go
index 5ff31c368..6954d5eae 100644
--- a/backend/internal/network/handler.go
+++ b/backend/internal/network/handler.go
@@ -14,7 +14,7 @@ import (
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
"github.com/samber/mo"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -285,12 +285,7 @@ func (h *NetworkHandler) DeleteNetwork(ctx context.Context, input *DeleteNetwork
return nil, huma.Error500InternalServerError("Failed to remove network: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Network removed successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Network removed successfully", activityID), nil
}
func (h *NetworkHandler) ConnectContainer(ctx context.Context, input *ConnectContainerInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -324,12 +319,7 @@ func (h *NetworkHandler) ConnectContainer(ctx context.Context, input *ConnectCon
return nil, huma.Error500InternalServerError("Failed to connect container to network: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Container connected successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Container connected successfully", activityID), nil
}
func (h *NetworkHandler) DisconnectContainer(ctx context.Context, input *DisconnectContainerInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -361,12 +351,7 @@ func (h *NetworkHandler) DisconnectContainer(ctx context.Context, input *Disconn
return nil, huma.Error500InternalServerError("Failed to disconnect container from network: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Container disconnected successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Container disconnected successfully", activityID), nil
}
func (h *NetworkHandler) PruneNetworks(ctx context.Context, input *PruneNetworksInput) (*handlerutil.Out[networktypes.PruneReport], error) {
diff --git a/backend/internal/notification/handler.go b/backend/internal/notification/handler.go
index d0fac7c41..0ffacc4b2 100644
--- a/backend/internal/notification/handler.go
+++ b/backend/internal/notification/handler.go
@@ -9,7 +9,7 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/notification"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -168,12 +168,7 @@ func (h *NotificationHandler) DeleteNotificationSettings(ctx context.Context, in
return nil, huma.Error500InternalServerError("Failed to delete notification settings: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Settings deleted successfully"},
- },
- }, nil
+ return handlerutil.MessageOutput("Settings deleted successfully", ""), nil
}
func (h *NotificationHandler) TestNotification(ctx context.Context, input *TestNotificationInput) (*handlerutil.Out[notification.TestResponse], error) {
diff --git a/backend/internal/passkey/handler.go b/backend/internal/passkey/handler.go
index 767494a62..964bdf1e9 100644
--- a/backend/internal/passkey/handler.go
+++ b/backend/internal/passkey/handler.go
@@ -369,7 +369,7 @@ func (h *PasskeyHandler) DeletePasskey(ctx context.Context, input *DeletePasskey
if deletePasskeyErr := h.passkeyService.DeletePasskey(ctx, userModel.ID, input.ID, sessionID, input.StepUpToken, oidcEnabled); deletePasskeyErr != nil {
return nil, passkeyHTTPErrorInternal(deletePasskeyErr)
}
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey deleted"}}}, nil
+ return handlerutil.MessageOutput("Passkey deleted", ""), nil
}
func (h *PasskeyHandler) BeginStepUp(ctx context.Context, _ *BeginStepUpInput) (*handlerutil.Out[passkeyBeginResponse], error) {
@@ -455,7 +455,7 @@ func (h *PasskeyHandler) DisableMFA(ctx context.Context, input *MFASettingsInput
return nil, passkeyHTTPErrorInternal(disableMFAErr)
}
h.authService.InvalidateUserTokenCache(userModel.ID)
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey MFA disabled"}}}, nil
+ return handlerutil.MessageOutput("Passkey MFA disabled", ""), nil
}
func (h *PasskeyHandler) RegenerateRecoveryCodes(ctx context.Context, input *MFASettingsInput) (*handlerutil.Out[RecoveryCodesResponse], error) {
diff --git a/backend/internal/project/handler.go b/backend/internal/project/handler.go
index 8b47f38f2..ac7ef2401 100644
--- a/backend/internal/project/handler.go
+++ b/backend/internal/project/handler.go
@@ -422,15 +422,7 @@ func (h *ProjectHandler) DownProject(ctx context.Context, input *DownProjectInpu
activitylib.FlushWriter(activityWriter)
activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Project stopped", nil)
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Project brought down successfully",
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Project brought down successfully", activityID), nil
}
func projectUpdateHTTPErrorInternal(err error) error {
@@ -687,15 +679,7 @@ func (h *ProjectHandler) DestroyProject(ctx context.Context, input *DestroyProje
activitylib.FlushWriter(activityWriter)
activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Project destroyed", nil)
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Project destroyed successfully",
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Project destroyed successfully", activityID), nil
}
// UpdateProject updates a Docker Compose project.
@@ -943,12 +927,7 @@ func (h *ProjectHandler) ArchiveProject(ctx context.Context, input *ArchiveProje
return nil, huma.Error500InternalServerError("Failed to archive project: " + archiveProjectErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Project archived successfully"},
- },
- }, nil
+ return handlerutil.MessageOutput("Project archived successfully", ""), nil
}
func (h *ProjectHandler) UnarchiveProject(ctx context.Context, input *UnarchiveProjectInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -965,12 +944,7 @@ func (h *ProjectHandler) UnarchiveProject(ctx context.Context, input *UnarchiveP
return nil, huma.Error500InternalServerError("Failed to unarchive project: " + unarchiveProjectErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Project unarchived successfully"},
- },
- }, nil
+ return handlerutil.MessageOutput("Project unarchived successfully", ""), nil
}
// PullProjectImages pulls all images for a project with streaming progress.
diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.go
index 8c0cbf86f..96a61aa93 100644
--- a/backend/internal/registry/handler.go
+++ b/backend/internal/registry/handler.go
@@ -8,7 +8,7 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -161,14 +161,7 @@ func (h *ContainerRegistryHandler) DeleteRegistry(ctx context.Context, input *De
h.triggerRemoteRegistrySync(ctx, "registry deletion")
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Container registry deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Container registry deleted successfully", ""), nil
}
// TestRegistry tests connectivity to a container registry.
@@ -184,14 +177,7 @@ func (h *ContainerRegistryHandler) TestRegistry(ctx context.Context, input *Test
if testECRRegistryErr := h.registryService.TestECRRegistry(ctx, reg); testECRRegistryErr != nil {
return nil, huma.Error400BadRequest("Registry test failed: " + testECRRegistryErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "ECR authentication succeeded",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("ECR authentication succeeded", ""), nil
}
decryptedToken, err := crypto.Decrypt(reg.Token)
@@ -206,14 +192,7 @@ func (h *ContainerRegistryHandler) TestRegistry(ctx context.Context, input *Test
noCredentials := strings.TrimSpace(reg.Username) == "" && strings.TrimSpace(decryptedToken) == ""
msg := kit.Ternary(noCredentials, "Registry saved (no credentials to test)", "Authentication succeeded")
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: msg,
- },
- },
- }, nil
+ return handlerutil.MessageOutput(msg, ""), nil
}
// SyncRegistries syncs container registries from a remote source.
@@ -223,14 +202,7 @@ func (h *ContainerRegistryHandler) SyncRegistries(ctx context.Context, input *Sy
return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to sync registries: "+err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Registries synced successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Registries synced successfully", ""), nil
}
func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Context, reason string) {
diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.go
index 86dc199e8..65bfef964 100644
--- a/backend/internal/registry/service.go
+++ b/backend/internal/registry/service.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "net"
"net/http"
"runtime"
"strings"
@@ -16,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/ecr"
"github.com/cenkalti/backoff/v5"
+ "github.com/containerd/errdefs"
"github.com/distribution/reference"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
@@ -23,6 +25,7 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/google/go-containerregistry/pkg/v1/remote/transport"
dockerregistry "github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
"github.com/samber/hot"
@@ -870,7 +873,7 @@ func (s *ContainerRegistryService) ImageDigest(ctx context.Context, imageRef str
func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {
parts, err := refs.NormalizeReference(imageRef)
if err != nil {
- return nil, err
+ return nil, common.Classify(common.ErrValidation, err)
}
var lastResult *containerregistry.DigestResult
@@ -933,15 +936,46 @@ func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, image
}, backoff.WithBackOff(bo), backoff.WithMaxTries(5))
if retryErr != nil {
- if errors.Is(retryErr, context.Canceled) || errors.Is(retryErr, context.DeadlineExceeded) {
+ if errors.Is(retryErr, context.Canceled) {
return lastResult, retryErr
}
- return lastResult, lastErr
+ if errors.Is(retryErr, context.DeadlineExceeded) {
+ return lastResult, common.Classify(common.ErrTimeout, retryErr)
+ }
+ return lastResult, classifyDigestErrorInternal(lastErr)
}
return lastResult, nil
}
+// classifyDigestErrorInternal tags digest lookup failures so callers can retry
+// transient ones. Retryable kinds are checked first because fallback errors are joined.
+func classifyDigestErrorInternal(err error) error {
+ if err == nil {
+ return nil
+ }
+ errLower := strings.ToLower(err.Error())
+ transportErr, isTransport := errors.AsType[*transport.Error](err)
+ var netErr net.Error
+ var dnsErr *net.DNSError
+ switch {
+ case errors.As(err, &dnsErr) && dnsErr.IsNotFound:
+ // A registry host that does not resolve is a bad reference, not an outage worth retrying.
+ return err
+ case isRateLimitErrorInternal(err), errdefs.IsUnavailable(err), errors.As(err, &netErr),
+ strings.Contains(errLower, "connection refused"), isTransport && transportErr.StatusCode >= http.StatusInternalServerError:
+ return common.Classify(common.ErrUnavailable, err)
+ case errors.Is(err, context.DeadlineExceeded), errdefs.IsDeadlineExceeded(err):
+ return common.Classify(common.ErrTimeout, err)
+ case browse.IsUnauthorizedRegistryError(err):
+ return common.Classify(common.ErrUnauthorized, err)
+ case errdefs.IsNotFound(err), isTransport && transportErr.StatusCode == http.StatusNotFound, strings.Contains(errLower, "manifest unknown"):
+ return common.Classify(common.ErrNotFound, err)
+ default:
+ return err
+ }
+}
+
func (
s *ContainerRegistryService,
) inspectImageDigestViaDaemonInternal(
diff --git a/backend/internal/registry/service_test.go b/backend/internal/registry/service_test.go
index 8e014e4c0..793e1c9e7 100644
--- a/backend/internal/registry/service_test.go
+++ b/backend/internal/registry/service_test.go
@@ -7,6 +7,7 @@ import (
"errors"
"io"
"log"
+ "net"
"net/http"
"net/http/httptest"
"net/url"
@@ -22,6 +23,7 @@ import (
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/random"
"github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/libtnb/sqlite"
"github.com/moby/moby/api/pkg/authconfig"
dockerregistry "github.com/moby/moby/api/types/registry"
@@ -1593,3 +1595,28 @@ func TestContainerRegistryService_ImageVersionLabelErrorNotCachedInternal(t *tes
require.NoError(t, err)
assert.Equal(t, "v2.8.0-next.67", label)
}
+
+func TestClassifyDigestError(t *testing.T) {
+ cases := map[string]struct {
+ err error
+ kind error
+ }{
+ "rate limit": {errors.New("toomanyrequests: rate limit exceeded"), common.ErrUnavailable},
+ "server error": {&transport.Error{StatusCode: http.StatusBadGateway}, common.ErrUnavailable},
+ "connection refused": {errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), common.ErrUnavailable},
+ "unauthorized": {&transport.Error{StatusCode: http.StatusUnauthorized}, common.ErrUnauthorized},
+ "not found": {&transport.Error{StatusCode: http.StatusNotFound}, common.ErrNotFound},
+ "manifest unknown": {errors.New("MANIFEST_UNKNOWN: manifest unknown"), common.ErrNotFound},
+ }
+ for name, tc := range cases {
+ require.ErrorIs(t, classifyDigestErrorInternal(tc.err), tc.kind, name)
+ }
+
+ plain := errors.New("x509: certificate signed by unknown authority")
+ classified := classifyDigestErrorInternal(plain)
+ require.Equal(t, plain, classified)
+ require.NotErrorIs(t, classified, common.ErrUnavailable)
+ // A host that does not resolve never comes back, so it must not be retried like an outage.
+ unknownHost := &url.Error{Op: "Get", URL: "https://invalid-registry.example.com/v2/", Err: &net.DNSError{Err: "no such host", Name: "invalid-registry.example.com", IsNotFound: true}}
+ require.NotErrorIs(t, classifyDigestErrorInternal(unknownHost), common.ErrUnavailable)
+}
diff --git a/backend/internal/role/handler.go b/backend/internal/role/handler.go
index ec0da38ee..48ee426c6 100644
--- a/backend/internal/role/handler.go
+++ b/backend/internal/role/handler.go
@@ -152,9 +152,7 @@ func (h *RoleHandler) DeleteRole(ctx context.Context, input *DeleteRoleInput) (*
}
return nil, huma.Error500InternalServerError("failed to delete role: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "role deleted"}},
- }, nil
+ return handlerutil.MessageOutput("role deleted", ""), nil
}
func (h *RoleHandler) GetPermissionsManifest(_ context.Context, _ *struct{}) (*handlerutil.Out[roletypes.PermissionsManifest], error) {
diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.go
index 5e4ebeac3..25be76263 100644
--- a/backend/internal/s3/handler.go
+++ b/backend/internal/s3/handler.go
@@ -158,7 +158,7 @@ func (h *s3DestinationHandlerInternal) deleteInternal(ctx context.Context, input
return nil, huma.Error500InternalServerError(err.Error())
}
h.triggerRemoteSyncInternal(ctx, "S3 destination deletion")
- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal("S3 destination deleted successfully")}, nil
+ return handlerutil.MessageOutput("S3 destination deleted successfully", ""), nil
}
func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *testS3DestinationInputInternal) (*handlerutil.Out[base.MessageResponse], error) {
@@ -168,11 +168,7 @@ func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *
}
return nil, huma.Error400BadRequest(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil
-}
-
-func successMessageResponseInternal(message string) base.ApiResponse[base.MessageResponse] {
- return base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message}}
+ return handlerutil.MessageOutput(s3ConnectionTestSuccessInternal, ""), nil
}
func (h *s3DestinationHandlerInternal) triggerRemoteSyncInternal(ctx context.Context, reason string) {
@@ -191,12 +187,12 @@ func (h *s3DestinationHandlerInternal) testConfigurationInternal(ctx context.Con
if err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil
+ return handlerutil.MessageOutput(s3ConnectionTestSuccessInternal, ""), nil
}
func (h *s3DestinationHandlerInternal) syncInternal(ctx context.Context, input *syncS3DestinationsInputInternal) (*handlerutil.Out[base.MessageResponse], error) {
if err := h.service.SyncS3Destinations(ctx, input.Body.Destinations); err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal("S3 destinations synced successfully")}, nil
+ return handlerutil.MessageOutput("S3 destinations synced successfully", ""), nil
}
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
index d0854e983..9812cbf27 100644
--- a/backend/internal/s3/service.go
+++ b/backend/internal/s3/service.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "log/slog"
"strings"
"github.com/getarcaneapp/arcane/types/v2/backup"
@@ -83,17 +84,22 @@ func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]bac
return s3DestinationsToDTOsInternal(destinations), nil
}
-// ListS3DestinationsByID indexes destination DTOs for backup metadata lookups.
-func (s *S3DestinationService) ListS3DestinationsByID(ctx context.Context) (map[string]backup.S3Destination, error) {
+// DestinationsByID indexes destinations to decorate backup records, such as with their names. Decoration never fails
+// its caller, so a missing service or failed listing yields an empty index.
+func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]backup.S3Destination {
+ if s == nil {
+ return nil
+ }
destinations, err := s.ListAllS3Destinations(ctx)
if err != nil {
- return nil, err
+ slog.WarnContext(ctx, "could not list S3 destinations", "error", err)
+ return nil
}
indexed := make(map[string]backup.S3Destination, len(destinations))
for _, destination := range destinations {
indexed[destination.ID] = destination
}
- return indexed, nil
+ return indexed
}
func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
index 27b494570..e4b6dc405 100644
--- a/backend/internal/s3/service_test.go
+++ b/backend/internal/s3/service_test.go
@@ -343,22 +343,19 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {
}
}
-func TestListS3DestinationsByIDInternal(t *testing.T) {
+func TestDestinationsByID(t *testing.T) {
service, db := setupS3DestinationServiceTestInternal(t)
- indexed, err := service.ListS3DestinationsByID(t.Context())
- require.NoError(t, err)
- require.Empty(t, indexed)
+ require.Empty(t, service.DestinationsByID(t.Context()))
destination := S3Destination{Name: "Offsite", Bucket: "backups", SecretAccessKey: "encrypted"}
require.NoError(t, db.Create(&destination).Error)
- indexed, err = service.ListS3DestinationsByID(t.Context())
- require.NoError(t, err)
+ indexed := service.DestinationsByID(t.Context())
require.Len(t, indexed, 1)
require.Equal(t, "Offsite", indexed[destination.ID].Name)
require.Equal(t, "backups", indexed[destination.ID].Bucket)
require.True(t, indexed[destination.ID].SecretConfigured)
require.Empty(t, indexed["missing"].Name)
+ // Listing failures and a missing service leave records undecorated rather than failing.
require.NoError(t, db.Migrator().DropTable(&S3Destination{}))
- indexed, err = service.ListS3DestinationsByID(t.Context())
- require.ErrorContains(t, err, "failed to list S3 destinations")
- require.Nil(t, indexed)
+ require.Nil(t, service.DestinationsByID(t.Context()))
+ require.Nil(t, (*S3DestinationService)(nil).DestinationsByID(t.Context()))
}
diff --git a/backend/internal/swarm/children/configs/handler.go b/backend/internal/swarm/children/configs/handler.go
index 782b7cc62..40866a4e7 100644
--- a/backend/internal/swarm/children/configs/handler.go
+++ b/backend/internal/swarm/children/configs/handler.go
@@ -133,5 +133,5 @@ func (h *Handler) DeleteConfig(ctx context.Context, input *DeleteSwarmConfigInpu
h.audit(ctx, input.EnvironmentID, "config.delete", "swarm_config", input.ConfigID, "", map[string]any{"configId": input.ConfigID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm config removed successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm config removed successfully", ""), nil
}
diff --git a/backend/internal/swarm/children/nodes/handler.go b/backend/internal/swarm/children/nodes/handler.go
index 8cf1d6ac1..ad8079f83 100644
--- a/backend/internal/swarm/children/nodes/handler.go
+++ b/backend/internal/swarm/children/nodes/handler.go
@@ -295,7 +295,7 @@ func (h *Handler) DeleteNodeAgentBinding(ctx context.Context, input *DeleteSwarm
if err := h.environmentService.DetachSwarmNodeEnvironment(ctx, input.EnvironmentID, input.NodeID); err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node environment detached"}}}, nil
+ return handlerutil.MessageOutput("Swarm node environment detached", ""), nil
}
// DeleteNodeAgentDeployment removes a dedicated hidden node-agent registration.
@@ -313,7 +313,7 @@ func (h *Handler) DeleteNodeAgentDeployment(ctx context.Context, input *DeleteSw
); deleteSwarmNodeAgentDeploymentErr != nil {
return nil, huma.Error500InternalServerError(deleteSwarmNodeAgentDeploymentErr.Error())
}
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Dedicated swarm node agent registration removed"}}}, nil
+ return handlerutil.MessageOutput("Dedicated swarm node agent registration removed", ""), nil
}
// GetNodeIdentity returns the swarm identity of the node serving the current request.
@@ -359,7 +359,7 @@ func (h *Handler) UpdateNode(ctx context.Context, input *UpdateSwarmNodeInput) (
h.audit(ctx, input.EnvironmentID, "node.update", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node updated successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm node updated successfully", ""), nil
}
// DeleteNode removes a swarm node from the cluster.
@@ -380,7 +380,7 @@ func (h *Handler) DeleteNode(ctx context.Context, input *DeleteSwarmNodeInput) (
h.audit(ctx, input.EnvironmentID, "node.delete", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID, "force": input.Force})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node removed successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm node removed successfully", ""), nil
}
// PromoteNode promotes a swarm worker to manager.
@@ -401,7 +401,7 @@ func (h *Handler) PromoteNode(ctx context.Context, input *PromoteSwarmNodeInput)
h.audit(ctx, input.EnvironmentID, "node.promote", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node promoted successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm node promoted successfully", ""), nil
}
// DemoteNode demotes a swarm manager to worker.
@@ -422,7 +422,7 @@ func (h *Handler) DemoteNode(ctx context.Context, input *DemoteSwarmNodeInput) (
h.audit(ctx, input.EnvironmentID, "node.demote", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node demoted successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm node demoted successfully", ""), nil
}
// ListNodeTasks lists tasks currently associated with a swarm node.
diff --git a/backend/internal/swarm/children/secrets/handler.go b/backend/internal/swarm/children/secrets/handler.go
index e50c080af..3d0c4fdad 100644
--- a/backend/internal/swarm/children/secrets/handler.go
+++ b/backend/internal/swarm/children/secrets/handler.go
@@ -133,5 +133,5 @@ func (h *Handler) DeleteSecret(ctx context.Context, input *DeleteSwarmSecretInpu
h.audit(ctx, input.EnvironmentID, "secret.delete", "swarm_secret", input.SecretID, "", map[string]any{"secretId": input.SecretID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm secret removed successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm secret removed successfully", ""), nil
}
diff --git a/backend/internal/swarm/children/services/handler.go b/backend/internal/swarm/children/services/handler.go
index 079302c04..19ea0f36d 100644
--- a/backend/internal/swarm/children/services/handler.go
+++ b/backend/internal/swarm/children/services/handler.go
@@ -191,7 +191,7 @@ func (h *Handler) DeleteService(ctx context.Context, input *DeleteSwarmServiceIn
h.audit(ctx, input.EnvironmentID, "service.delete", "swarm_service", input.ServiceID, "", map[string]any{"serviceId": input.ServiceID})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm service removed successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm service removed successfully", ""), nil
}
// ListServiceTasks lists tasks belonging to a specific swarm service.
diff --git a/backend/internal/swarm/children/stacks/handler.go b/backend/internal/swarm/children/stacks/handler.go
index bb7f72bdc..651a3e7e4 100644
--- a/backend/internal/swarm/children/stacks/handler.go
+++ b/backend/internal/swarm/children/stacks/handler.go
@@ -217,7 +217,7 @@ func (h *Handler) DeleteStack(ctx context.Context, input *DeleteSwarmStackInput)
h.audit(ctx, input.EnvironmentID, "stack.delete", "swarm_stack", input.Name, input.Name, map[string]any{"stack": input.Name})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm stack removed successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm stack removed successfully", ""), nil
}
// ListStackServices lists services belonging to a swarm stack.
diff --git a/backend/internal/swarm/handler.go b/backend/internal/swarm/handler.go
index 161d7f0cf..997445b9d 100644
--- a/backend/internal/swarm/handler.go
+++ b/backend/internal/swarm/handler.go
@@ -198,7 +198,7 @@ func (h *SwarmHandler) JoinSwarm(ctx context.Context, input *JoinSwarmInput) (*h
h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.join", "swarm", "cluster", "cluster", map[string]any{"remoteAddrs": input.Body.RemoteAddrs})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Joined swarm successfully"}}}, nil
+ return handlerutil.MessageOutput("Joined swarm successfully", ""), nil
}
// LeaveSwarm removes the target engine from its current swarm cluster.
@@ -219,7 +219,7 @@ func (h *SwarmHandler) LeaveSwarm(ctx context.Context, input *LeaveSwarmInput) (
h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.leave", "swarm", "cluster", "cluster", map[string]any{"force": input.Body.Force})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Left swarm successfully"}}}, nil
+ return handlerutil.MessageOutput("Left swarm successfully", ""), nil
}
// UnlockSwarm unlocks a swarm manager using the supplied unlock key.
@@ -240,7 +240,7 @@ func (h *SwarmHandler) UnlockSwarm(ctx context.Context, input *UnlockSwarmInput)
h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.unlock", "swarm", "cluster", "cluster", map[string]any{})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm unlocked successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm unlocked successfully", ""), nil
}
// GetUnlockKey returns the current swarm manager unlock key.
@@ -310,7 +310,7 @@ func (h *SwarmHandler) RotateJoinTokens(ctx context.Context, input *RotateSwarmJ
},
)
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm join tokens rotated successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm join tokens rotated successfully", ""), nil
}
// UpdateSwarmSpec updates the swarm cluster specification.
@@ -331,7 +331,7 @@ func (h *SwarmHandler) UpdateSwarmSpec(ctx context.Context, input *UpdateSwarmSp
h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.update_spec", "swarm", "cluster", "cluster", map[string]any{})
- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm spec updated successfully"}}}, nil
+ return handlerutil.MessageOutput("Swarm spec updated successfully", ""), nil
}
// auditSwarmMutation writes an informational event for a completed swarm mutation.
diff --git a/backend/internal/system/children/backup/children/volumes/service.go b/backend/internal/system/children/backup/children/volumes/service.go
index 91db944de..ba9047407 100644
--- a/backend/internal/system/children/backup/children/volumes/service.go
+++ b/backend/internal/system/children/backup/children/volumes/service.go
@@ -6,76 +6,147 @@ import (
"errors"
"fmt"
"log/slog"
+ "path"
"slices"
"strings"
+ "time"
"uuid"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
- "go.getarcane.app/kit/pkg"
+ "github.com/italypaleale/francis/builtin/workflow"
+ kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
-// Dependencies are the services and system admission hooks volume backups run on.
+const (
+ systemVolumeBackupConfigKey = "systemVolumeBackupConfig"
+ systemVolumeBackupJobPrefix = "volumes:"
+ // systemVolumePlanTarget records a scheduled run's frozen selection.
+ systemVolumePlanTarget = "system-volume-plan"
+ defaultSystemVolumeSchedule = "0 0 2 * * *"
+)
+
+// Dependencies are the services and system admission volume backups run on.
type Dependencies struct {
- DB *database.DB
- Engine *backup.Engine
- Volumes *volume.VolumeService
- S3Destinations *s3.S3DestinationService
- Activity *activity.ActivityService
- Settings *settings.SettingsService
- Jobs *entityjobs.Registry
- AcquireRun func(ctx context.Context) (*runs.Lease, error)
- AcquireDurableRun func(ctx context.Context, runID string) (*runs.Lease, error)
- AlreadyRunning error
+ DB *database.DB
+ Engine *backup.Engine
+ Volumes *volume.VolumeService
+ S3Destinations *s3.S3DestinationService
+ Activity *activity.ActivityService
+ Settings *settings.SettingsService
+ Jobs *entityjobs.Registry
+ AlreadyRunning error
}
// Service owns system-managed volume backup policies and their runs.
type Service struct {
- db *database.DB
- engine *backup.Engine
- volumes *volume.VolumeService
- s3Destinations *s3.S3DestinationService
- activity *activity.ActivityService
- settings *settings.SettingsService
- jobs *entityjobs.Registry
- acquireRun func(ctx context.Context) (*runs.Lease, error)
- acquireDurableRun func(ctx context.Context, runID string) (*runs.Lease, error)
- alreadyRunning error
+ db *database.DB
+ engine *backup.Engine
+ volumes *volume.VolumeService
+ s3Destinations *s3.S3DestinationService
+ activity *activity.ActivityService
+ settings *settings.SettingsService
+ jobs *entityjobs.Registry
+ alreadyRunning error
+ flow *flow.Engine
+ // backupWorkflow runs the selections Start accepted; policyWorkflow runs scheduled policies.
+ backupWorkflow *flow.Workflow
+ policyWorkflow *flow.Workflow
+}
+
+// selectionInput freezes the policy and volumes one run backs up; a manual run also carries its requester.
+type selectionInput struct {
+ Policy backuptypes.SystemVolumeBackupPolicy `json:"policy"`
+ ManualPolicy bool `json:"manualPolicy"`
+ Candidates []backuptypes.SystemVolumeBackupOption `json:"candidates"`
+ Requester *backuptypes.Requester `json:"requester,omitempty"`
+ // Claim names the manual run that parked the system admission for its own task.
+ Claim string `json:"claim,omitempty"`
}
func NewService(deps Dependencies) *Service {
return &Service{
- db: deps.DB,
- engine: deps.Engine,
- volumes: deps.Volumes,
- s3Destinations: deps.S3Destinations,
- activity: deps.Activity,
- settings: deps.Settings,
- jobs: deps.Jobs,
- acquireRun: deps.AcquireRun,
- acquireDurableRun: deps.AcquireDurableRun,
- alreadyRunning: deps.AlreadyRunning,
+ db: deps.DB,
+ engine: deps.Engine,
+ volumes: deps.Volumes,
+ s3Destinations: deps.S3Destinations,
+ activity: deps.Activity,
+ settings: deps.Settings,
+ jobs: deps.Jobs,
+ alreadyRunning: deps.AlreadyRunning,
+ }
+}
+
+// RegisterWorkflows defines the manual and scheduled volume backup workflows while the host is still unstarted.
+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
+ var err error
+ s.flow = engine
+ template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceID: new("volumes"), ResourceName: new("Volumes")}
+ s.backupWorkflow, err = engine.Define(flow.Definition{
+ Name: "system-volume-backup",
+ Version: 1,
+ Fingerprint: "9b4a5e46a984446bc08df39a71e563e031548db3af1825a2413e8c96515db6ab",
+ Concurrency: 1,
+ Timeout: 24 * time.Hour,
+ Activity: template,
+ Labels: map[string]string{"backup": "Backing up volumes"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var input selectionInput
+ if decodeErr := t.Payload(&input); decodeErr != nil {
+ return nil, decodeErr
+ }
+ result, backupErr := s.backUpSelection(ctx, input, volume.VolumeBackupTriggerManual, t.ActivityID())
+ if backupErr != nil {
+ return nil, backupErr
+ }
+ return selectionOutcome(result, false), nil
+ }), workflow.WithMaxAttempts(1))},
+ })
+ if err != nil {
+ return err
}
+ template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Backing up volumes", "Creating scheduled system-managed volume backups"
+ s.policyWorkflow, err = engine.Define(flow.Definition{
+ Name: "system-volume-backup-policy",
+ Version: 1,
+ Fingerprint: "94a13d1bac530e6eddcb73c27990a07d69915dc947ce5b559166eb3fb85e39ef",
+ Concurrency: 1,
+ Timeout: 24 * time.Hour,
+ Activity: template,
+ Labels: map[string]string{"backup": "Backing up volumes"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ previous, _ := jobcontext.Run(ctx)
+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == systemVolumePlanTarget }) {
+ return s.resumeSelection(ctx, previous, t.ActivityID())
+ }
+ var policyID string
+ if decodeErr := t.Payload(&policyID); decodeErr != nil {
+ return nil, decodeErr
+ }
+ return s.runScheduledSystemVolumeBackup(ctx, policyID, t.ActivityID())
+ }), workflow.WithMaxAttempts(1))},
+ })
+ return err
}
-func (s *Service) loadSystemVolumeBackupPoliciesInternal() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {
+func (s *Service) loadSystemVolumeBackupPolicies() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {
collection := &backuptypes.SystemVolumeBackupPolicyCollection{Policies: []backuptypes.SystemVolumeBackupPolicy{}}
if s.settings == nil {
return collection, nil
@@ -93,31 +164,24 @@ func (s *Service) loadSystemVolumeBackupPoliciesInternal() (*backuptypes.SystemV
return collection, nil
}
-func (s *Service) systemVolumeBackupPolicyInternal(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {
- collection, err := s.loadSystemVolumeBackupPoliciesInternal()
+func (s *Service) systemVolumeBackupPolicy(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {
+ collection, err := s.loadSystemVolumeBackupPolicies()
if err != nil {
return nil, err
}
- for i := range collection.Policies {
- if collection.Policies[i].ID == policyID {
- policy := collection.Policies[i]
- return &policy, nil
- }
+ index := slices.IndexFunc(collection.Policies, func(policy backuptypes.SystemVolumeBackupPolicy) bool { return policy.ID == policyID })
+ if index < 0 {
+ return nil, nil
}
- return nil, nil
+ return &collection.Policies[index], nil
}
func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBackupPolicyCollection, error) {
- collection, err := s.loadSystemVolumeBackupPoliciesInternal()
+ collection, err := s.loadSystemVolumeBackupPolicies()
if err != nil {
return nil, err
}
- destinations := make(map[string]backuptypes.S3Destination)
- if s.s3Destinations != nil {
- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {
- destinations = available
- }
- }
+ destinations := s.s3Destinations.DestinationsByID(ctx)
for i := range collection.Policies {
policy := &collection.Policies[i]
policy.S3DestinationName = destinations[policy.S3DestinationID].Name
@@ -143,9 +207,10 @@ func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBacku
return collection, nil
}
-func (s *Service) normalizeSystemVolumePolicyUpdateInternal(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {
- if err := validateSystemVolumeSelectionInternal(input.SelectionMode); err != nil {
- return backuptypes.SystemVolumeBackupPolicy{}, err
+func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {
+ mode := input.SelectionMode
+ if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {
+ return backuptypes.SystemVolumeBackupPolicy{}, errors.New("selectionMode must be all, allowlist, or blocklist")
}
update, err := backup.ValidatePolicyUpdate(ctx, "system-managed volume", input.UpdateBackupPolicy, s.s3Destinations)
if err != nil {
@@ -153,13 +218,13 @@ func (s *Service) normalizeSystemVolumePolicyUpdateInternal(ctx context.Context,
}
names := kit.Unique(kit.TrimNonEmpty(input.VolumeNames))
slices.Sort(names)
- if input.SelectionMode == backuptypes.SystemVolumeSelectionAll || names == nil {
+ if mode == backuptypes.SystemVolumeSelectionAll || names == nil {
names = []string{}
}
return backuptypes.SystemVolumeBackupPolicy{
ID: input.ID, Enabled: update.Enabled, Schedule: update.Schedule, RetentionCount: update.RetentionCount,
StopContainers: update.StopContainers, LocalEnabled: update.LocalEnabled, S3Enabled: update.S3Enabled,
- S3DestinationID: update.S3DestinationID, SelectionMode: input.SelectionMode,
+ S3DestinationID: update.S3DestinationID, SelectionMode: mode,
VolumeNames: names, IgnoreAnonymous: input.IgnoreAnonymous,
}, nil
}
@@ -168,7 +233,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
if s.settings == nil {
return nil, errors.New("settings service is unavailable")
}
- existing, err := s.loadSystemVolumeBackupPoliciesInternal()
+ existing, err := s.loadSystemVolumeBackupPolicies()
if err != nil {
return nil, err
}
@@ -181,7 +246,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
return backuptypes.SystemVolumeBackupPolicy{ID: uuid.New().String()}
},
Build: func(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy, update backuptypes.UpdateSystemVolumeBackupPolicy) error {
- normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdateInternal(ctx, update)
+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, update)
if normalizeErr != nil {
return normalizeErr
}
@@ -189,11 +254,17 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update
*policy = normalized
return nil
},
- Persist: s.saveSystemVolumeBackupPoliciesInternal,
+ Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {
+ encoded, encodeErr := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})
+ if encodeErr != nil {
+ return fmt.Errorf("encode policies: %w", encodeErr)
+ }
+ return s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))
+ },
Unregister: func(ctx context.Context, policyID string) {
s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)
},
- Reschedule: s.rescheduleSystemVolumeBackupInternal,
+ Reschedule: s.rescheduleSystemVolumeBackup,
}
if runErr := reconcile.Run(ctx, updates); runErr != nil {
return nil, runErr
@@ -210,7 +281,7 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBa
if err != nil {
return nil, err
}
- collection, err := s.loadSystemVolumeBackupPoliciesInternal()
+ collection, err := s.loadSystemVolumeBackupPolicies()
if err != nil {
return nil, err
}
@@ -232,152 +303,132 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBa
return options, nil
}
-func (s *Service) resolveSystemVolumeRunPolicyInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (backuptypes.SystemVolumeBackupPolicy, bool, error) {
- if request.PolicyID != "" {
- if request.Custom != nil {
- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("select a saved policy or custom configuration, not both")
- }
- policy, err := s.systemVolumeBackupPolicyInternal(request.PolicyID)
+// prepareSystemVolumeBackups resolves the requested policy, takes the system admission, and selects the live volumes.
+func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (selectionInput, *runs.Lease, error) {
+ if s.volumes == nil {
+ return selectionInput{}, nil, errors.New("volume service is unavailable")
+ }
+ var input selectionInput
+ switch {
+ case request.PolicyID != "" && request.Custom != nil:
+ return selectionInput{}, nil, errors.New("select a saved policy or custom configuration, not both")
+ case request.PolicyID != "":
+ policy, err := s.systemVolumeBackupPolicy(request.PolicyID)
if err != nil {
- return backuptypes.SystemVolumeBackupPolicy{}, false, err
+ return selectionInput{}, nil, err
}
if policy == nil {
- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("system-managed volume backup policy not found")
+ return selectionInput{}, nil, errors.New("system-managed volume backup policy not found")
}
- return *policy, false, nil
- }
- custom := request.Custom
- if custom != nil &&
- custom.Destination != backuptypes.SystemBackupDestinationLocal &&
- custom.Destination != backuptypes.SystemBackupDestinationS3 &&
- custom.Destination != backuptypes.SystemBackupDestinationLocalS3 {
- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("destination must be local, s3, or local_s3")
- }
- policy, err := s.normalizeSystemVolumePolicyUpdateInternal(ctx, customSystemVolumePolicyInternal(custom))
- return policy, true, err
-}
-
-type preparedSystemVolumeBackupInternal struct {
- policy backuptypes.SystemVolumeBackupPolicy
- manualPolicy bool
- candidates []backuptypes.SystemVolumeBackupOption
- lease *runs.Lease
-}
-
-func (
- s *Service,
-) runSystemVolumeBackupsInternal(
- ctx context.Context,
- request backuptypes.RunSystemVolumeBackupsRequest,
- trigger volume.VolumeBackupTrigger,
-) (
- *backuptypes.SystemVolumeBackupRunResult,
- error,
-) {
- prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)
- if err != nil {
- return nil, err
+ input.Policy = *policy
+ default:
+ custom := backuptypes.UpdateSystemVolumeBackupPolicy{
+ Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,
+ SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,
+ }
+ if run := request.Custom; run != nil {
+ localEnabled := run.Destination == backuptypes.SystemBackupDestinationLocal || run.Destination == backuptypes.SystemBackupDestinationLocalS3
+ s3Enabled := run.Destination == backuptypes.SystemBackupDestinationS3 || run.Destination == backuptypes.SystemBackupDestinationLocalS3
+ if !localEnabled && !s3Enabled {
+ return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")
+ }
+ custom = backuptypes.UpdateSystemVolumeBackupPolicy{
+ Enabled: true, Schedule: defaultSystemVolumeSchedule, StopContainers: run.StopContainers,
+ LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: run.S3DestinationID,
+ SelectionMode: run.SelectionMode, VolumeNames: run.VolumeNames, IgnoreAnonymous: run.IgnoreAnonymous,
+ }
+ }
+ policy, err := s.normalizeSystemVolumePolicyUpdate(ctx, custom)
+ if err != nil {
+ return selectionInput{}, nil, err
+ }
+ input.Policy, input.ManualPolicy = policy, true
}
- defer prepared.lease.Release(ctx)
- frozen, err := json.Marshal(manualSystemVolumesInternal{Policy: prepared.policy, ManualPolicy: prepared.manualPolicy, Candidates: prepared.candidates})
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)
if err != nil {
- return nil, err
- }
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_plan",
- ID: "system-volume-plan",
- Status: scheduler.Succeeded,
- RecoveryData: frozen,
- },
- ); progressErr != nil {
- return nil, progressErr
- }
- return s.executeSystemVolumeBackupsInternal(ctx, prepared.policy, prepared.manualPolicy, prepared.candidates, trigger, "")
-}
-
-func (s *Service) prepareSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*preparedSystemVolumeBackupInternal, error) {
- if s.volumes == nil {
- return nil, errors.New("volume service is unavailable")
+ return selectionInput{}, nil, err
}
- policyConfig, manualPolicy, err := s.resolveSystemVolumeRunPolicyInternal(ctx, request)
- if err != nil {
- return nil, err
- }
- lease, err := s.acquireRun(ctx)
- if err != nil {
- return nil, err
+ if !admitted {
+ return selectionInput{}, nil, s.alreadyRunning
}
options, err := s.volumes.ListBackupVolumeOptions(ctx)
if err != nil {
lease.Release(ctx)
- return nil, err
+ return selectionInput{}, nil, err
}
- candidates := selectSystemVolumeBackupCandidatesInternal(policyConfig, options)
- return &preparedSystemVolumeBackupInternal{policy: policyConfig, manualPolicy: manualPolicy, candidates: candidates, lease: lease}, nil
+ input.Candidates = selectSystemVolumeBackupCandidates(input.Policy, options)
+ return input, lease, nil
}
-func (
- s *Service,
-) executeSystemVolumeBackupsInternal(
- ctx context.Context,
- policyConfig backuptypes.SystemVolumeBackupPolicy,
- manualPolicy bool,
- candidates []backuptypes.SystemVolumeBackupOption,
- trigger volume.VolumeBackupTrigger,
- activityID string,
-) (
- result *backuptypes.SystemVolumeBackupRunResult,
- err error,
-) {
- result = &backuptypes.SystemVolumeBackupRunResult{
- Matched: len(candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0),
- }
- defer s.updateSystemVolumeProgressInternal(context.WithoutCancel(ctx), activityID, policyConfig.ID, candidates, result)
+func (s *Service) executeSystemVolumeBackups(
+ ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string,
+) (result *backuptypes.SystemVolumeBackupRunResult, err error) {
+ result = &backuptypes.SystemVolumeBackupRunResult{Matched: len(input.Candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0)}
+ reportProgress := func(ctx context.Context) {
+ if activityID == "" {
+ return
+ }
+ names := make([]string, len(input.Candidates))
+ for i, candidate := range input.Candidates {
+ names[i] = candidate.Name
+ }
+ progress := 100
+ if result.Matched > 0 {
+ progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched
+ }
+ _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{
+ "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names,
+ "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,
+ }})
+ if updateErr != nil {
+ slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", input.Policy.ID, "error", updateErr)
+ }
+ }
+ defer reportProgress(context.WithoutCancel(ctx))
defer utils.RecoverToError(&err, "system-managed volume backup")
policy := backuptypes.UpdateBackupPolicy{
- Enabled: true, Schedule: policyConfig.Schedule, RetentionCount: policyConfig.RetentionCount,
- StopContainers: policyConfig.StopContainers, LocalEnabled: policyConfig.LocalEnabled, S3Enabled: policyConfig.S3Enabled,
- S3DestinationID: policyConfig.S3DestinationID,
+ Enabled: true, Schedule: input.Policy.Schedule, RetentionCount: input.Policy.RetentionCount,
+ StopContainers: input.Policy.StopContainers, LocalEnabled: input.Policy.LocalEnabled, S3Enabled: input.Policy.S3Enabled,
+ S3DestinationID: input.Policy.S3DestinationID,
}
- for _, candidate := range candidates {
- status := completedVolumeBackupStatusInternal(ctx, candidate.Name)
- if status == scheduler.Succeeded {
+ previous, _ := jobcontext.Run(ctx)
+ for _, candidate := range input.Candidates {
+ done := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == candidate.Name && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)
+ })
+ if done >= 0 && previous.Outcome.Targets[done].Status == scheduler.Succeeded {
result.Succeeded++
continue
}
- if status == scheduler.Skipped {
+ if done >= 0 {
result.Skipped++
continue
}
if cancellationErr := ctx.Err(); cancellationErr != nil {
return result, cancellationErr
}
- s.updateSystemVolumeProgressInternal(ctx, activityID, policyConfig.ID, candidates, result)
- overridden, policyErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)
- if policyErr != nil {
- result.Failed++
- result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: policyErr.Error()})
- continue
- }
- if overridden {
+ reportProgress(ctx)
+ overridden, backupErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)
+ if backupErr == nil && overridden {
if skipProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Skipped}); skipProgressErr != nil {
return result, skipProgressErr
}
result.Skipped++
continue
}
- seriesID := systemVolumePolicyIDInternal(policyConfig.ID, candidate.Name)
- if manualPolicy {
- seriesID = systemVolumeManualPolicyIDInternal(candidate.Name)
+ if backupErr == nil {
+ seriesID := backuptypes.SystemVolumePolicyPrefix + kit.Ternary(input.ManualPolicy, "manual", input.Policy.ID) + ":" + kit.SHA256Hex(candidate.Name)[:16]
+ _, backupErr = s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)
}
- _, backupErr := s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)
if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {
result.Skipped++
continue
}
+ if backupErr != nil && ctx.Err() != nil {
+ // Shutdown interrupted this volume; hand the task back instead of counting it failed.
+ return result, ctx.Err()
+ }
if backupErr != nil {
result.Failed++
result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})
@@ -391,279 +442,124 @@ func (
return result, nil
}
-func (s *Service) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {
- policy, err := s.systemVolumeBackupPolicyInternal(policyID)
+func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID, activityID string) (scheduler.Outcome, error) {
+ policy, err := s.systemVolumeBackupPolicy(policyID)
if err != nil {
return scheduler.Outcome{}, err
}
if policy == nil || !policy.Enabled {
return scheduler.Outcome{Status: scheduler.Skipped}, nil
}
- remoteDisabled, checkErr := s.disableMissingVolumeS3Internal(ctx, policy)
- if checkErr != nil {
- return scheduler.Outcome{}, checkErr
+ var remoteErr error
+ if policy.S3Enabled {
+ root := path.Join(backup.VolumeRoot, s.settings.GetSettingsConfig().InstanceID.Value)
+ remoteErr = backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)
+ }
+ remoteDisabled, err := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {
+ collection, loadErr := s.loadSystemVolumeBackupPolicies()
+ if loadErr != nil {
+ return false, loadErr
+ }
+ index := slices.IndexFunc(collection.Policies, func(current backuptypes.SystemVolumeBackupPolicy) bool {
+ return current.ID == policy.ID && current.S3DestinationID == policy.S3DestinationID && current.S3Enabled == policy.S3Enabled &&
+ current.Enabled == policy.Enabled && current.LocalEnabled == policy.LocalEnabled
+ })
+ if index < 0 {
+ return false, nil
+ }
+ current := &collection.Policies[index]
+ *kit.Ternary(field == "enabled", &current.Enabled, &current.S3Enabled) = false
+ encoded, encodeErr := json.Marshal(collection)
+ if encodeErr != nil {
+ return false, fmt.Errorf("encode policies: %w", encodeErr)
+ }
+ return true, s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))
+ })
+ if err != nil {
+ return scheduler.Outcome{}, err
+ }
+ if remoteDisabled {
+ s.rescheduleSystemVolumeBackup(ctx, policy)
}
if remoteDisabled && !policy.LocalEnabled {
return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
}
- result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)
+ input, lease, err := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})
if errors.Is(err, s.alreadyRunning) {
slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)
- return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: a system backup is running"}, nil
+ }
+ var result *backuptypes.SystemVolumeBackupRunResult
+ if err == nil {
+ defer lease.Release(ctx)
+ var frozen []byte
+ if frozen, err = json.Marshal(input); err == nil {
+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_plan", ID: systemVolumePlanTarget, Status: scheduler.Succeeded, RecoveryData: frozen})
+ }
+ if err == nil {
+ result, err = s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)
+ }
}
if err != nil {
slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)
return scheduler.Outcome{}, err
}
- slog.InfoContext(
- ctx,
- "Scheduled system-managed volume backups completed",
- "policyId",
- policyID,
- "matched",
- result.Matched,
- "succeeded",
- result.Succeeded,
- "failed",
- result.Failed,
- "skipped",
- result.Skipped,
- )
- outcome := scheduler.Outcome{Status: scheduler.Succeeded}
- if remoteDisabled {
- outcome.Message = backup.RemoteDisabledMessage
- }
- if result.Failed > 0 || remoteDisabled {
- outcome.Status = scheduler.Partial
- }
- for _, failure := range result.Failures {
- outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})
- }
- return outcome, nil
+ slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID,
+ "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)
+ return selectionOutcome(result, remoteDisabled), nil
}
-func (s *Service) rescheduleSystemVolumeBackupInternal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {
- if policy == nil {
- return
- }
- jobID := systemVolumeBackupJobPrefix + policy.ID
- if !policy.Enabled {
- s.jobs.Unregister(ctx, jobID)
- return
- }
- policyID := policy.ID
- s.jobs.Register(ctx, jobID, func(context.Context) string {
- current, err := s.systemVolumeBackupPolicyInternal(policyID)
- if err != nil || current == nil {
- return defaultSystemVolumeSchedule
- }
- return current.Schedule
- }, func(ctx context.Context) (scheduler.Outcome, error) {
- return s.runScheduledSystemVolumeBackupInternal(ctx, policyID)
- }, func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
- for _, target := range previous.Outcome.Targets {
- if target.ID != "system-volume-plan" || len(target.RecoveryData) == 0 {
- continue
- }
- if err := s.ExecuteDurable(ctx, previous.ID, target.RecoveryData, true); err != nil {
- return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: err.Error()}, err
- }
- return scheduler.Outcome{Status: scheduler.Succeeded}, nil
- }
- return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "The interrupted volume backup has no frozen selection"}, nil
+// resumeSelection finishes the frozen selection an interrupted scheduled run recorded.
+func (s *Service) resumeSelection(ctx context.Context, previous scheduler.Run, activityID string) (scheduler.Outcome, error) {
+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == systemVolumePlanTarget && len(target.RecoveryData) > 0
})
-}
-
-func (s *Service) saveSystemVolumeBackupPoliciesInternal(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {
- encoded, err := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})
- if err != nil {
- return fmt.Errorf("encode policies: %w", err)
- }
- return s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))
-}
-
-func (s *Service) disableMissingVolumeS3Internal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) (bool, error) {
- if !policy.S3Enabled {
- return false, nil
+ if index < 0 {
+ return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "The interrupted volume backup has no frozen selection"}, nil
}
- root := "arcane-volume-backups/" + s.settings.GetSettingsConfig().InstanceID.Value
- err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)
- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
- return false, nil
+ var input selectionInput
+ if err := json.Unmarshal(previous.Outcome.Targets[index].RecoveryData, &input); err != nil {
+ return scheduler.Outcome{}, err
}
- collection, err := s.loadSystemVolumeBackupPoliciesInternal()
+ result, err := s.backUpSelection(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)
if err != nil {
- return false, err
+ return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: err.Error()}, err
}
- for i := range collection.Policies {
- current := &collection.Policies[i]
- if current.ID != policy.ID {
- continue
- }
- if current.S3DestinationID != policy.S3DestinationID || current.S3Enabled != policy.S3Enabled || current.Enabled != policy.Enabled || current.LocalEnabled != policy.LocalEnabled {
- return false, nil
- }
- if current.LocalEnabled {
- current.S3Enabled = false
- } else {
- current.Enabled = false
- }
- if saveSystemVolumeBackupPoliciesErr := s.saveSystemVolumeBackupPoliciesInternal(ctx, collection.Policies); saveSystemVolumeBackupPoliciesErr != nil {
- return false, saveSystemVolumeBackupPoliciesErr
- }
- *policy = *current
- s.rescheduleSystemVolumeBackupInternal(ctx, policy)
- return true, nil
- }
- return false, nil
+ return selectionOutcome(result, false), nil
}
-// Start freezes the selected policy and volumes before returning.
-func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {
- prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)
+// backUpSelection takes over the system admission, resumes the selection's interrupted volumes, and backs up the rest.
+func (s *Service) backUpSelection(ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string) (*backuptypes.SystemVolumeBackupRunResult, error) {
+ lease, admitted, err := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim)
if err != nil {
return nil, err
}
- policy, manualPolicy, candidates, lease := prepared.policy, prepared.manualPolicy, prepared.candidates, prepared.lease
-
- names := make([]string, len(candidates))
- for i, candidate := range candidates {
- names[i] = candidate.Name
- }
- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activity, "0", activitytypes.TypeResourceAction, "system_backup", "volumes", "Volumes", &user,
- "Backing up volumes", "Creating system-managed volume backups", database.JSON{
- "action": "run_system_volume_backups",
- "policyId": policy.ID,
- "volumeNames": names,
- "matched": len(
- candidates,
- ),
- "succeeded": 0,
- "failed": 0,
- "skipped": 0,
- "failures": []backuptypes.SystemVolumeBackupFailure{},
- }, false)
- if activityID == "" {
- lease.Release(ctx)
- return nil, errors.New("failed to create system-managed volume backup activity")
- }
- finish := func(runErr error) {
- defer lease.Release(ctx)
- activitylib.CompleteHandlerActivity(workCtx, s.activity, activityID, "System-managed volume backups completed", runErr)
- }
- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
- payload, err := json.Marshal(manualSystemVolumesInternal{Policy: policy, ManualPolicy: manualPolicy, Candidates: candidates, ActivityID: activityID, UserID: user.ID})
- if err == nil {
- err = s.engine.SubmitDurableRun(
- workCtx,
- backuptypes.DurableRunCommand{
- Kind: "system-volumes",
- RunID: activityID,
- ActivityID: activityID,
- Payload: payload,
- UserID: user.ID,
- EnvironmentID: "0",
- Permission: authz.PermSystemBackupsManage,
- RequestedWithKey: keyID,
- },
- lease,
- )
- }
-
- if err != nil {
- finish(err)
- return nil, err
- }
- return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil
-}
-
-func (
- s *Service,
-) updateSystemVolumeProgressInternal(
- ctx context.Context,
- activityID, policyID string,
- candidates []backuptypes.SystemVolumeBackupOption,
- result *backuptypes.SystemVolumeBackupRunResult,
-) {
- if activityID == "" {
- return
- }
- names := make([]string, len(candidates))
- for i, candidate := range candidates {
- names[i] = candidate.Name
- }
- progress := 100
- if result.Matched > 0 {
- progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched
- }
- _, err := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{
- "action": "run_system_volume_backups", "policyId": policyID, "volumeNames": names,
- "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,
- }})
- if err != nil {
- slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", policyID, "error", err)
- }
-}
-
-type manualSystemVolumesInternal struct {
- Policy backuptypes.SystemVolumeBackupPolicy `json:"policy"`
- ManualPolicy bool `json:"manualPolicy"`
- Candidates []backuptypes.SystemVolumeBackupOption `json:"candidates"`
- ActivityID string `json:"activityId"`
- UserID string `json:"userId"`
-}
-
-// ExecuteDurable runs a submitted or interrupted system-volumes run.
-func (s *Service) ExecuteDurable(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
- var command manualSystemVolumesInternal
- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {
- return decodeErr
- }
- defer func() {
- if ctx.Err() == nil {
- activitylib.CompleteHandlerActivity(ctx, s.activity, command.ActivityID, "System-managed volume backups completed", err)
- }
- }()
- lease, err := s.acquireDurableRun(ctx, runID)
- if err != nil {
- return err
+ if !admitted {
+ return nil, s.alreadyRunning
}
defer lease.Release(ctx)
- if interrupted {
- ctx, err = s.reconcileVolumeCandidatesInternal(ctx, command.Candidates)
- if err != nil {
- return err
+ if input.Requester != nil {
+ if authorizeErr := s.engine.Authorize(ctx, *input.Requester); authorizeErr != nil {
+ return nil, authorizeErr
}
}
- result, err := s.executeSystemVolumeBackupsInternal(ctx, command.Policy, command.ManualPolicy, command.Candidates, volume.VolumeBackupTriggerManual, command.ActivityID)
- if err != nil {
- return err
- }
- if result.Failed > 0 {
- return fmt.Errorf("%d volume backups failed", result.Failed)
- }
- return nil
-}
-
-func (s *Service) reconcileVolumeCandidatesInternal(ctx context.Context, candidates []backuptypes.SystemVolumeBackupOption) (context.Context, error) {
previous, _ := jobcontext.Run(ctx)
- for _, candidate := range candidates {
- pending := false
- for _, target := range previous.Outcome.Targets {
- if target.ID == candidate.Name && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped {
- pending = true
- break
- }
- }
- if !pending {
+ for _, candidate := range input.Candidates {
+ if !slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == candidate.Name && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped
+ }) {
continue
}
outcome, recoveryErr := s.volumes.ReconcileBackup(ctx, previous, candidate.Name)
+ if errors.Is(recoveryErr, backup.ErrBackupSettled) {
+ // That volume's backup already failed, so the selection backs it up again below.
+ continue
+ }
if recoveryErr != nil {
- return ctx, recoveryErr
+ return nil, recoveryErr
}
if outcome.Status != scheduler.Succeeded {
- return ctx, errors.New(outcome.Message)
+ return nil, errors.New(outcome.Message)
}
for i := range previous.Outcome.Targets {
if previous.Outcome.Targets[i].ID == candidate.Name {
@@ -673,86 +569,109 @@ func (s *Service) reconcileVolumeCandidatesInternal(ctx context.Context, candida
}
progressCtx := ctx
ctx = jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(progressCtx, target) })
- return ctx, nil
+ return s.executeSystemVolumeBackups(ctx, input, trigger, activityID)
}
-// FailDurable completes the activity of a system-volumes run that cannot continue.
-func (s *Service) FailDurable(ctx context.Context, _ string, payload []byte, runErr error) error {
- var command manualSystemVolumesInternal
- if err := json.Unmarshal(payload, &command); err != nil {
- return err
+// selectionOutcome reports a run's result; failed volumes or disabled remote storage make it partial.
+func selectionOutcome(result *backuptypes.SystemVolumeBackupRunResult, remoteDisabled bool) scheduler.Outcome {
+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "System-managed volume backups completed"}
+ if result.Failed > 0 {
+ outcome.Status, outcome.Message = scheduler.Partial, fmt.Sprintf("%d volume backups failed", result.Failed)
}
- activitylib.CompleteHandlerActivity(ctx, s.activity, command.ActivityID, "System-managed volume backups completed", runErr)
- return nil
-}
-
-// RegisterJobsOnStartup schedules every saved policy and returns how many were loaded.
-func (s *Service) RegisterJobsOnStartup(ctx context.Context) int {
- policies, err := s.loadSystemVolumeBackupPoliciesInternal()
- if err != nil {
- slog.ErrorContext(ctx, "Failed to load system-managed volume backup policies", "error", err)
- return 0
+ if remoteDisabled {
+ outcome.Status, outcome.Message = scheduler.Partial, backup.RemoteDisabledMessage
}
- for i := range policies.Policies {
- s.rescheduleSystemVolumeBackupInternal(ctx, &policies.Policies[i])
+ for _, failure := range result.Failures {
+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})
}
- return len(policies.Policies)
+ return outcome
}
-func completedVolumeBackupStatusInternal(ctx context.Context, volumeName string) scheduler.RunStatus {
- previous, ok := jobcontext.Run(ctx)
- if !ok {
- return ""
+func (s *Service) rescheduleSystemVolumeBackup(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {
+ if policy == nil {
+ return
}
- for _, target := range previous.Outcome.Targets {
- if target.ID == volumeName && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped) {
- return target.Status
- }
+ jobID := systemVolumeBackupJobPrefix + policy.ID
+ if !policy.Enabled {
+ s.jobs.Unregister(ctx, jobID)
+ return
}
- return ""
+ policyID := policy.ID
+ s.jobs.Add(ctx, &flow.Job{
+ Engine: s.flow,
+ Workflow: s.policyWorkflow,
+ JobName: s.jobs.JobName(jobID),
+ Payload: policyID,
+ Activity: activitylib.StartRequest{Metadata: database.JSON{"action": "scheduled_system_volume_backups", "policyId": policyID, "schedule": policy.Schedule}},
+ ScheduleFn: func(context.Context) string {
+ current, err := s.systemVolumeBackupPolicy(policyID)
+ if err != nil || current == nil {
+ return defaultSystemVolumeSchedule
+ }
+ return current.Schedule
+ },
+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
+ return s.resumeSelection(ctx, previous, "")
+ },
+ })
}
-func customSystemVolumePolicyInternal(custom *backuptypes.SystemVolumeBackupCustomRun) backuptypes.UpdateSystemVolumeBackupPolicy {
- if custom == nil {
- return backuptypes.UpdateSystemVolumeBackupPolicy{
- Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,
- SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,
- }
+// Start freezes the selected policy and volumes, then submits their workflow.
+func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {
+ requester := backup.NewRequester(ctx, user, "0", authz.PermSystemBackupsManage)
+ if err := s.engine.Authorize(ctx, requester); err != nil {
+ return nil, err
}
- localEnabled := custom.Destination == backuptypes.SystemBackupDestinationLocal || custom.Destination == backuptypes.SystemBackupDestinationLocalS3
- s3Enabled := custom.Destination == backuptypes.SystemBackupDestinationS3 || custom.Destination == backuptypes.SystemBackupDestinationLocalS3
- return backuptypes.UpdateSystemVolumeBackupPolicy{
- Enabled: true, Schedule: defaultSystemVolumeSchedule, RetentionCount: 0, StopContainers: custom.StopContainers,
- LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: custom.S3DestinationID,
- SelectionMode: custom.SelectionMode, VolumeNames: custom.VolumeNames, IgnoreAnonymous: custom.IgnoreAnonymous,
+ input, lease, err := s.prepareSystemVolumeBackups(ctx, request)
+ if err != nil {
+ return nil, err
}
+ input.Requester = &requester
+ names := make([]string, len(input.Candidates))
+ for i, candidate := range input.Candidates {
+ names[i] = candidate.Name
+ }
+ input.Claim = uuid.NewV7().String()
+ release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim, lease)
+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{
+ StartedBy: &user, Step: "Backing up volumes", LatestMessage: "Creating system-managed volume backups",
+ Metadata: database.JSON{
+ "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names, "matched": len(names),
+ "succeeded": 0, "failed": 0, "skipped": 0, "failures": []backuptypes.SystemVolumeBackupFailure{},
+ },
+ })
+ if err != nil {
+ release(ctx)
+ return nil, err
+ }
+ // A run canceled before its task starts never takes the parked lease.
+ go func() {
+ _, _ = s.flow.Wait(context.WithoutCancel(ctx), s.backupWorkflow, activityID)
+ release(context.WithoutCancel(ctx))
+ }()
+ return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil
}
-const (
- systemVolumeBackupConfigKey = "systemVolumeBackupConfig"
- systemVolumeBackupJobPrefix = "volumes:"
- defaultSystemVolumeSchedule = "0 0 2 * * *"
-)
-
-func systemVolumeManualPolicyIDInternal(volumeName string) string {
- return backuptypes.SystemVolumePolicyPrefix + "manual:" + kit.SHA256Hex(volumeName)[:16]
-}
-
-func systemVolumePolicyIDInternal(policyID, volumeName string) string {
- return backuptypes.SystemVolumePolicyPrefix + policyID + ":" + kit.SHA256Hex(volumeName)[:16]
+// RegisterJobsOnStartup schedules every saved policy and returns how many were loaded.
+func (s *Service) RegisterJobsOnStartup(ctx context.Context) int {
+ policies, err := s.loadSystemVolumeBackupPolicies()
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to load system-managed volume backup policies", "error", err)
+ return 0
+ }
+ for i := range policies.Policies {
+ s.rescheduleSystemVolumeBackup(ctx, &policies.Policies[i])
+ }
+ return len(policies.Policies)
}
-func selectSystemVolumeBackupCandidatesInternal(policy backuptypes.SystemVolumeBackupPolicy, options []backuptypes.SystemVolumeBackupOption) []backuptypes.SystemVolumeBackupOption {
- configured := make(map[string]struct{}, len(policy.VolumeNames))
- for _, name := range policy.VolumeNames {
- configured[name] = struct{}{}
- }
+func selectSystemVolumeBackupCandidates(policy backuptypes.SystemVolumeBackupPolicy, options []backuptypes.SystemVolumeBackupOption) []backuptypes.SystemVolumeBackupOption {
result := make([]backuptypes.SystemVolumeBackupOption, 0, len(options))
for _, option := range options {
if !option.Available {
continue
}
- _, selected := configured[option.Name]
+ selected := slices.Contains(policy.VolumeNames, option.Name)
matches := policy.SelectionMode == backuptypes.SystemVolumeSelectionAll ||
(policy.SelectionMode == backuptypes.SystemVolumeSelectionAllowlist && selected) ||
(policy.SelectionMode == backuptypes.SystemVolumeSelectionBlocklist && !selected)
@@ -762,10 +681,3 @@ func selectSystemVolumeBackupCandidatesInternal(policy backuptypes.SystemVolumeB
}
return result
}
-
-func validateSystemVolumeSelectionInternal(mode backuptypes.SystemVolumeSelectionMode) error {
- if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {
- return errors.New("selectionMode must be all, allowlist, or blocklist")
- }
- return nil
-}
diff --git a/backend/internal/system/children/backup/children/volumes/service_test.go b/backend/internal/system/children/backup/children/volumes/service_test.go
index e844504b5..7770e30cd 100644
--- a/backend/internal/system/children/backup/children/volumes/service_test.go
+++ b/backend/internal/system/children/backup/children/volumes/service_test.go
@@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/require"
)
-func TestSystemVolumeBackupSelectionInternal(t *testing.T) {
+func TestSystemVolumeBackupSelection(t *testing.T) {
options := []backup.SystemVolumeBackupOption{
{Name: "app", Available: true},
{Name: "cache", Available: true},
@@ -37,7 +37,7 @@ func TestSystemVolumeBackupSelectionInternal(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- selected := selectSystemVolumeBackupCandidatesInternal(tt.config, options)
+ selected := selectSystemVolumeBackupCandidates(tt.config, options)
names := make([]string, len(selected))
for i := range selected {
names[i] = selected[i].Name
diff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.go
index 87f25f56e..c98d526f1 100644
--- a/backend/internal/system/children/backup/handler.go
+++ b/backend/internal/system/children/backup/handler.go
@@ -3,13 +3,11 @@ package backup
import (
"context"
"errors"
- "strings"
"github.com/danielgtaylor/huma/v2"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/base"
- "github.com/samber/mo"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -248,7 +246,7 @@ func (h *Handler) Restore(ctx context.Context, input *RestoreSystemBackupInput)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return messageOutputInternal("Arcane system restore started", activityID), nil
+ return handlerutil.MessageOutput("Arcane system restore started", activityID), nil
}
// BrowseFiles returns one lazy-loaded project tree page.
@@ -285,7 +283,7 @@ func (h *Handler) RestoreFiles(ctx context.Context, input *RestoreSystemBackupFi
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return messageOutputInternal("Arcane project files restored successfully", activityID), nil
+ return handlerutil.MessageOutput("Arcane project files restored successfully", activityID), nil
}
func (h *Handler) Upload(ctx context.Context, input *UploadSystemBackupInput) (*SystemBackupOutput, error) {
@@ -324,21 +322,5 @@ func (h *Handler) Delete(ctx context.Context, input *DeleteSystemBackupInput) (*
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return messageOutputInternal("Arcane system backup deleted successfully", activityID), nil
-}
-
-func messageOutputInternal(message, activityID string) *handlerutil.Out[base.MessageResponse] {
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: message,
- ActivityID: mo.EmptyableToOption(
- strings.TrimSpace(
- activityID,
- ),
- ).ToPointer(),
- },
- },
- }
+ return handlerutil.MessageOutput("Arcane system backup deleted successfully", activityID), nil
}
diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.go
index 46047cefb..797219455 100644
--- a/backend/internal/system/children/backup/service.go
+++ b/backend/internal/system/children/backup/service.go
@@ -23,14 +23,14 @@ import (
"github.com/getarcaneapp/arcane/types/v2/recovery"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
- "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -39,7 +39,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup/children/snapshots"
@@ -50,6 +49,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
@@ -59,7 +59,6 @@ import (
const (
defaultSystemBackupSchedule = "0 0 3 * * *"
systemRecoveryHelperPath = "/app/arcane-recovery-helper"
- systemAdmissionID = "system"
)
var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")
@@ -88,7 +87,7 @@ type Store struct {
unregister func(context.Context, string),
reschedule func(context.Context, *backuptypes.SystemBackupPolicy),
) error
- DisablePolicyRemote func(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error)
+ DisablePolicyRemote func(ctx context.Context, policy *backuptypes.SystemBackupPolicy, field string) (bool, error)
CheckScheduledRemote func(ctx context.Context, destinationID string) error
ListHistory func(ctx context.Context, params pagination.QueryParams) ([]backuptypes.HistoryEntry, pagination.Response, error)
}
@@ -136,6 +135,29 @@ type Service struct {
jobs *entityjobs.Registry
snapshots *snapshots.Service
volumes *volumes.Service
+ flow *flow.Engine
+ // backupWorkflow runs backups StartBackup accepted; policyWorkflow runs scheduled policies.
+ backupWorkflow *flow.Workflow
+ policyWorkflow *flow.Workflow
+}
+
+// manualBackupInput is a manual backup's workflow payload; the run row is created before it starts.
+type manualBackupInput struct {
+ Checkpoint systemBackupRecovery `json:"checkpoint"`
+ Requester backuptypes.Requester `json:"requester"`
+}
+
+// systemBackupRecovery is the checkpoint a run records so an interrupted backup can resume.
+type systemBackupRecovery struct {
+ BackupID string `json:"backupId"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+}
+
+type preparedSystemBackup struct {
+ run *backuptypes.SystemBackupRun
+ recoveryKey string
+ checkpoint systemBackupRecovery
}
func NewService(deps Dependencies) *Service {
@@ -149,78 +171,29 @@ func NewService(deps Dependencies) *Service {
deps.Engine,
deps.SQLDB,
deps.ActorStore,
- service.localRepositoryInternal,
- service.remoteRepositoryInternal,
- service.databaseFileInternal,
- service.projectsDirectoryInternal,
- service.recoveryEnvironmentInternal,
+ service.localRepository,
+ service.remoteRepository,
+ service.databaseFile,
+ func(ctx context.Context) string {
+ return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")
+ },
+ service.recoveryEnvironment,
)
service.volumes = volumes.NewService(volumes.Dependencies{
- DB: deps.DB,
- Engine: deps.Engine,
- Volumes: deps.Volumes,
- S3Destinations: deps.S3Destinations,
- Activity: deps.Activity,
- Settings: deps.Settings,
- Jobs: service.jobs,
- AcquireRun: service.acquireRunInternal,
- AcquireDurableRun: service.acquireDurableRunInternal,
- AlreadyRunning: ErrSystemBackupAlreadyRunning,
+ DB: deps.DB,
+ Engine: deps.Engine,
+ Volumes: deps.Volumes,
+ S3Destinations: deps.S3Destinations,
+ Activity: deps.Activity,
+ Settings: deps.Settings,
+ Jobs: service.jobs,
+ AlreadyRunning: ErrSystemBackupAlreadyRunning,
})
- if deps.Engine != nil {
- deps.Engine.RegisterRunKind("system", service.executeDurableBackupInternal, service.failDurableBackupInternal)
- deps.Engine.RegisterRunKind("system-volumes", service.volumes.ExecuteDurable, service.volumes.FailDurable)
- }
return service
}
-// acquireRunInternal takes the shared system backup admission lease.
-func (s *Service) acquireRunInternal(ctx context.Context) (*runs.Lease, error) {
- lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)
- if err != nil {
- return nil, err
- }
- if !admitted {
- return nil, ErrSystemBackupAlreadyRunning
- }
- return lease, nil
-}
-
-// acquireDurableRunInternal takes the shared system backup admission lease for a durable run.
-func (s *Service) acquireDurableRunInternal(ctx context.Context, runID string) (*runs.Lease, error) {
- lease, admitted, err := s.engine.AcquireDurableRun(ctx, runID, backup.SystemAdmissionScope, systemAdmissionID)
- if err != nil {
- return nil, err
- }
- if !admitted {
- return nil, ErrSystemBackupAlreadyRunning
- }
- return lease, nil
-}
-
-// SupportedDatabaseProvider reports whether system recovery works on the
-// configured database. Recovery is currently SQLite-only.
-func (s *Service) SupportedDatabaseProvider() bool {
- return strings.HasPrefix(s.config.DatabaseURL, "file:")
-}
-
-func recoveryHelperExecutableInternal(mounts []container.MountPoint, executablePath string) (string, *mount.Mount, error) {
- executablePath = strings.TrimSpace(executablePath)
- if executablePath == "" {
- return "", nil, errors.New("arcane executable path is empty")
- }
- executableMount := docker.MountForSubpath(mounts, executablePath, systemRecoveryHelperPath)
- if executableMount == nil {
- return executablePath, nil, nil
- }
- if executableMount.Type != mount.TypeBind {
- return "", nil, errors.New("the Arcane executable must come from the image or a bind mount")
- }
- executableMount.ReadOnly = true
- return systemRecoveryHelperPath, executableMount, nil
-}
-
-func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]string {
+// recoveryEnvironment is the runtime configuration, secrets included, that a restored instance needs.
+func (s *Service) recoveryEnvironment(ctx context.Context) map[string]string {
result := make(map[string]string)
var visit func(reflect.Value)
visit = func(value reflect.Value) {
@@ -233,10 +206,8 @@ func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]st
if value.Kind() != reflect.Struct {
return
}
- typeOf := value.Type()
for i := range value.NumField() {
- fieldType := typeOf.Field(i)
- field := value.Field(i)
+ fieldType, field := value.Type().Field(i), value.Field(i)
if fieldType.Anonymous {
visit(field)
continue
@@ -245,26 +216,88 @@ func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]st
if name == "" || !field.CanInterface() {
continue
}
- if field.Type() == reflect.TypeFor[os.FileMode]() {
- mode, ok := reflect.TypeAssert[os.FileMode](field)
- if ok {
- result[name] = fmt.Sprintf("%#o", uint32(mode))
- }
+ if mode, ok := reflect.TypeAssert[os.FileMode](field); ok {
+ result[name] = fmt.Sprintf("%#o", uint32(mode))
} else {
result[name] = fmt.Sprint(field.Interface())
}
}
}
visit(reflect.ValueOf(s.config))
- if value := s.projectsSettingInternal(ctx); value != "" {
+ if value := s.projectsSetting(ctx); value != "" {
result["PROJECTS_DIRECTORY"] = value
}
return result
}
-// databaseFileInternal resolves the SQLite database file: under /app/data in
+// acquireRun takes the shared system backup admission lease.
+func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)
+ if err != nil {
+ return nil, err
+ }
+ if !admitted {
+ return nil, ErrSystemBackupAlreadyRunning
+ }
+ return lease, nil
+}
+
+// RegisterWorkflows defines the manual and scheduled backup workflows while the host is still unstarted.
+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
+ var err error
+ s.flow = engine
+ template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceName: new("Arcane")}
+ s.backupWorkflow, err = engine.Define(flow.Definition{
+ Name: "system-backup",
+ Version: 1,
+ Fingerprint: "9b44e4c38b7fcf8951468cc43a51b66c644d1104f1a56fa49bf8c065dc51c046",
+ Concurrency: 1,
+ Timeout: 24 * time.Hour,
+ Activity: template,
+ Labels: map[string]string{"backup": "Creating system backup"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},
+ })
+ if err != nil {
+ return err
+ }
+ template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Creating scheduled system backup", "Creating scheduled Arcane system backup"
+ s.policyWorkflow, err = engine.Define(flow.Definition{
+ Name: "system-backup-policy",
+ Version: 1,
+ Fingerprint: "42fa9de20fefac192148a5f6d68132e34ae87948160b5bb0cc5876d506843bcf",
+ Concurrency: 1,
+ Timeout: 24 * time.Hour,
+ Activity: template,
+ Labels: map[string]string{"backup": "Creating scheduled system backup"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var policyID string
+ if decodeErr := t.Payload(&policyID); decodeErr != nil {
+ return nil, decodeErr
+ }
+ previous, _ := jobcontext.Run(ctx)
+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == policyID && len(target.RecoveryData) > 0
+ }) {
+ return s.resumeScheduledBackup(ctx, previous, policyID)
+ }
+ return s.runScheduledBackup(ctx, policyID)
+ }), workflow.WithMaxAttempts(1))},
+ })
+ if err != nil {
+ return err
+ }
+ return s.volumes.RegisterWorkflows(engine)
+}
+
+// SupportedDatabaseProvider reports whether system recovery works on the
+// configured database. Recovery is currently SQLite-only.
+func (s *Service) SupportedDatabaseProvider() bool {
+ return strings.HasPrefix(s.config.DatabaseURL, "file:")
+}
+
+// databaseFile resolves the SQLite database file: under /app/data in
// the container, the local data directory in host development.
-func (s *Service) databaseFileInternal() (string, error) {
+func (s *Service) databaseFile() (string, error) {
databasePath, err := kit.SQLitePathFromDSN(s.config.DatabaseURL)
if err != nil {
return "", fmt.Errorf("failed to parse database URL: %w", err)
@@ -279,17 +312,7 @@ func (s *Service) databaseFileInternal() (string, error) {
return absolute, nil
}
-// dataDirectoryInternal resolves the directory holding the SQLite database.
-// The recovery manifest must live inside it so every snapshot includes it.
-func (s *Service) dataDirectoryInternal() (string, error) {
- databaseFile, err := s.databaseFileInternal()
- if err != nil {
- return "", err
- }
- return filepath.Dir(databaseFile), nil
-}
-
-func (s *Service) localRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
+func (s *Service) localRepository(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
storage, err := s.volumeService.BackupStorageMount(ctx, dockerClient, "/repository", readOnly)
if err != nil {
return backup.Repository{}, err
@@ -301,7 +324,7 @@ func (s *Service) localRepositoryInternal(ctx context.Context, dockerClient *cli
}, nil
}
-func (s *Service) remoteRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {
+func (s *Service) remoteRepository(ctx context.Context, destinationID string) (backup.Repository, error) {
configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
if err != nil {
return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")
@@ -312,7 +335,7 @@ func (s *Service) remoteRepositoryInternal(ctx context.Context, destinationID st
}, nil
}
-func (s *Service) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {
+func (s *Service) recoveryKey(ctx context.Context, supplied string) (string, error) {
if strings.TrimSpace(supplied) != "" {
if err := backup.ValidateRecoveryKey(supplied); err != nil {
return "", err
@@ -329,299 +352,172 @@ func (s *Service) recoveryKeyInternal(ctx context.Context, supplied string) (str
return key, nil
}
-// resolveBackupPlanInternal resolves which destinations a run targets, from
-// either the policy or the request's destination enum.
-func (
- s *Service,
-) resolveBackupPlanInternal(
- ctx context.Context,
- request backuptypes.CreateSystemBackupRequest,
-) (
- localEnabled, s3Enabled bool,
- destinationID string,
- destination backuptypes.SystemBackupDestination,
- err error,
-) {
- destinationID = strings.TrimSpace(request.S3DestinationID)
+func (s *Service) CreateBackup(ctx context.Context, user usertypes.Actor, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
+ lease, err := s.acquireRun(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer lease.Release(ctx)
+ prepared, err := s.prepareBackup(ctx, trigger, request)
+ if err != nil {
+ return nil, err
+ }
+ return s.executeBackup(ctx, prepared)
+}
+
+// prepareBackup resolves the run's destinations from its policy or destination enum, creates the run row, and records its checkpoint.
+func (s *Service) prepareBackup(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackup, error) {
+ if !s.SupportedDatabaseProvider() {
+ return nil, errors.New("arcane system recovery currently requires the SQLite database provider")
+ }
+ recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)
+ if err != nil {
+ return nil, err
+ }
+ var checkpoint systemBackupRecovery
+ destinationID := strings.TrimSpace(request.S3DestinationID)
if request.PolicyID != "" {
policy, policyErr := s.store.Policy(ctx, request.PolicyID)
if policyErr != nil {
- return false, false, "", "", policyErr
+ return nil, policyErr
}
if policy == nil {
- return false, false, "", "", errors.New("system backup policy not found")
+ return nil, errors.New("system backup policy not found")
}
- localEnabled, s3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
+ checkpoint.LocalEnabled, checkpoint.S3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
} else {
switch request.Destination {
case backuptypes.SystemBackupDestinationLocal, "":
- localEnabled = true
+ checkpoint.LocalEnabled = true
case backuptypes.SystemBackupDestinationS3:
- s3Enabled = true
+ checkpoint.S3Enabled = true
case backuptypes.SystemBackupDestinationLocalS3:
- localEnabled, s3Enabled = true, true
+ checkpoint.LocalEnabled, checkpoint.S3Enabled = true, true
default:
- return false, false, "", "", errors.New("unknown system backup destination")
+ return nil, errors.New("unknown system backup destination")
}
}
- if !localEnabled && !s3Enabled {
- return false, false, "", "", errors.New("select at least one system backup destination")
+ if !checkpoint.LocalEnabled && !checkpoint.S3Enabled {
+ return nil, errors.New("select at least one system backup destination")
}
- if s3Enabled && destinationID == "" {
- return false, false, "", "", errors.New("select an S3 destination for the system backup")
+ if checkpoint.S3Enabled && destinationID == "" {
+ return nil, errors.New("select an S3 destination for the system backup")
}
- destination = backuptypes.SystemBackupDestinationLocal
- if localEnabled && s3Enabled {
+ destination := backuptypes.SystemBackupDestinationS3
+ switch {
+ case checkpoint.LocalEnabled && checkpoint.S3Enabled:
destination = backuptypes.SystemBackupDestinationLocalS3
- } else if s3Enabled {
- destination = backuptypes.SystemBackupDestinationS3
- }
- return localEnabled, s3Enabled, destinationID, destination, nil
-}
-
-func (s *Service) CreateBackup(ctx context.Context, user usertypes.Actor, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
- lease, err := s.acquireRunInternal(ctx)
- if err != nil {
- return nil, err
- }
- defer lease.Release(ctx)
- return s.createBackupInternal(ctx, trigger, request)
-}
-
-func (
- s *Service,
-) createStagedSystemRecoverySnapshotInternal(
- ctx context.Context,
- dockerClient *client.Client,
- recoveryKey, backupID, destinationID string,
- localEnabled, s3Enabled bool,
-) (
- backup.Snapshot,
- backup.Repository,
- bool,
- error,
-) {
- localRepository, err := s.localRepositoryInternal(ctx, dockerClient, false)
- var snapshot backup.Snapshot
- if err == nil {
- snapshot, err = s.snapshots.Create(ctx, dockerClient, localRepository, recoveryKey, backupID)
- }
- if err == nil {
- return snapshot, localRepository, false, nil
- }
- localStagingErr := fmt.Errorf("failed to create local system recovery staging snapshot: %w", err)
- if localEnabled || !s3Enabled {
- return backup.Snapshot{}, backup.Repository{}, false, localStagingErr
- }
- // S3-only backups normally use local repository staging. If local storage is unusable, write directly to
- // S3 so remote backups and restore safety snapshots remain available.
- remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)
- if repoErr != nil {
- return backup.Snapshot{}, backup.Repository{}, false, errors.Join(localStagingErr, repoErr)
- }
- remoteSnapshot, snapshotErr := s.snapshots.Create(ctx, dockerClient, remoteRepository, recoveryKey, backupID)
- if snapshotErr != nil {
- return backup.Snapshot{}, backup.Repository{}, false, errors.Join(localStagingErr, fmt.Errorf("failed to create S3 system recovery snapshot: %w", snapshotErr))
- }
- return remoteSnapshot, remoteRepository, true, nil
-}
-
-type systemBackupRecoveryInternal struct {
- BackupID string `json:"backupId"`
- LocalEnabled bool `json:"localEnabled"`
- S3Enabled bool `json:"s3Enabled"`
-}
-
-type preparedSystemBackupInternal struct {
- run *backuptypes.SystemBackupRun
- recoveryKey string
- localEnabled bool
- s3Enabled bool
-}
-
-func (s *Service) createBackupInternal(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
- prepared, err := s.prepareBackupInternal(ctx, trigger, request)
- if err != nil {
- return nil, err
- }
- return s.executeBackupInternal(ctx, prepared)
-}
-
-func (s *Service) prepareBackupInternal(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackupInternal, error) {
- if !s.SupportedDatabaseProvider() {
- return nil, errors.New("arcane system recovery currently requires the SQLite database provider")
- }
- recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
- if err != nil {
- return nil, err
- }
- localEnabled, s3Enabled, destinationID, destination, err := s.resolveBackupPlanInternal(ctx, request)
- if err != nil {
- return nil, err
+ case checkpoint.LocalEnabled:
+ destination = backuptypes.SystemBackupDestinationLocal
}
run := &backuptypes.SystemBackupRun{
- CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,
+ ID: "system-" + uuid.New().String(), CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,
Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID,
}
- run.ID = "system-" + uuid.New().String()
- if createBackupRunErr := s.store.CreateRun(ctx, run); createBackupRunErr != nil {
- return nil, createBackupRunErr
+ if createErr := s.store.CreateRun(ctx, run); createErr != nil {
+ return nil, createErr
}
- checkpoint, err := json.Marshal(systemBackupRecoveryInternal{BackupID: run.ID, LocalEnabled: localEnabled, S3Enabled: s3Enabled})
+ checkpoint.BackupID = run.ID
+ recoveryData, err := json.Marshal(checkpoint)
if err == nil {
- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: checkpoint})
+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: recoveryData})
}
if err != nil {
return nil, err
}
- return &preparedSystemBackupInternal{run: run, recoveryKey: recoveryKey, localEnabled: localEnabled, s3Enabled: s3Enabled}, nil
+ return &preparedSystemBackup{run: run, recoveryKey: recoveryKey, checkpoint: checkpoint}, nil
}
-func (s *Service) executeBackupInternal(ctx context.Context, prepared *preparedSystemBackupInternal) (_ *backuptypes.SystemBackupRun, err error) {
- run, recoveryKey := prepared.run, prepared.recoveryKey
- localEnabled, s3Enabled, destinationID := prepared.localEnabled, prepared.s3Enabled, run.S3DestinationID
-
- defer func() { err = s.completeBackupInternal(ctx, run, err) }()
+// executeBackup stages one snapshot, keeps it locally and/or replicates it to S3, then records the run's result.
+func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBackup) (_ *backuptypes.SystemBackupRun, err error) {
+ run, key, checkpoint := prepared.run, prepared.recoveryKey, prepared.checkpoint
+ defer func() {
+ if err = cmp.Or(err, ctx.Err()); err != nil {
+ run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()
+ } else {
+ run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
+ }
+ if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {
+ err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
+ }
+ if err == nil {
+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})
+ }
+ }()
defer utils.RecoverToError(&err, "system backup")
+ progress := func(stage string, status scheduler.RunStatus, message string) error {
+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":" + stage, Status: status, Message: message})
+ }
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return run, err
}
- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":stage", Status: scheduler.Running}); progressErr != nil {
+ if progressErr := progress("stage", scheduler.Running, ""); progressErr != nil {
return run, progressErr
}
- stagedSnapshot, stagingRepository, directRemote, err := s.createStagedSystemRecoverySnapshotInternal(
- ctx, dockerClient, recoveryKey, run.ID, destinationID, localEnabled, s3Enabled,
- )
- if err != nil {
- return run, err
- }
- if startDestinationProgressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_destination",
- ID: run.ID + ":stage",
- Status: scheduler.Succeeded,
- Message: stagedSnapshot.ID,
- },
- ); startDestinationProgressErr != nil {
- return run, startDestinationProgressErr
+ local, err := s.localRepository(ctx, dockerClient, false)
+ var staged backup.Snapshot
+ if err == nil {
+ staged, err = s.snapshots.Create(ctx, dockerClient, local, key, run.ID)
}
+ directRemote := err != nil
if directRemote {
- run.RemoteSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size
- if saveDisabledBackupErr := s.store.SaveRun(ctx, run); saveDisabledBackupErr != nil {
- return run, saveDisabledBackupErr
- }
- return run, nil
- }
- localRepository := stagingRepository
- // Both destinations use the same staged database copy.
- // The staged snapshot is kept only for local-enabled runs. Forgetting it in
- // a defer covers the failure paths too: a failed replication otherwise
- // orphans one unreferenced snapshot in the staging repository per run.
- defer func() {
- if localEnabled {
- return
+ stagingErr := fmt.Errorf("failed to create local system recovery staging snapshot: %w", err)
+ if checkpoint.LocalEnabled || !checkpoint.S3Enabled {
+ return run, stagingErr
}
- cleanupCtx := context.WithoutCancel(ctx)
- if forgetErr := s.engine.ForgetSnapshots(cleanupCtx, dockerClient, localRepository, recoveryKey, []string{stagedSnapshot.ID}); forgetErr != nil {
- slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshotId", stagedSnapshot.ID, "error", forgetErr)
+ // S3-only backups write straight to S3 when local staging storage is unusable.
+ direct, repoErr := s.remoteRepository(ctx, run.S3DestinationID)
+ if repoErr != nil {
+ return run, errors.Join(stagingErr, repoErr)
}
- }()
- if localEnabled {
- run.LocalSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size
- if saveLocalBackupErr := s.store.SaveRun(ctx, run); saveLocalBackupErr != nil {
- return run, saveLocalBackupErr
- }
- if localDestinationProgressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_destination",
- ID: run.ID + ":local",
- Status: scheduler.Succeeded,
- Message: stagedSnapshot.ID,
- },
- ); localDestinationProgressErr != nil {
- return run, localDestinationProgressErr
+ if staged, err = s.snapshots.Create(ctx, dockerClient, direct, key, run.ID); err != nil {
+ return run, errors.Join(stagingErr, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err))
}
}
- if s3Enabled {
- if replicateBackupErr := s.replicateBackupInternal(ctx, dockerClient, prepared, localRepository, stagedSnapshot); replicateBackupErr != nil {
- return run, replicateBackupErr
- }
+ if progressErr := progress("stage", scheduler.Succeeded, staged.ID); progressErr != nil {
+ return run, progressErr
}
- return run, nil
-}
-
-func (s *Service) completeBackupInternal(ctx context.Context, run *backuptypes.SystemBackupRun, err error) error {
- if err == nil {
- err = ctx.Err()
+ if directRemote {
+ run.RemoteSnapshotID, run.Size = staged.ID, staged.Size
+ return run, s.store.SaveRun(ctx, run)
}
- if err != nil {
- run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()
+ if checkpoint.LocalEnabled {
+ run.LocalSnapshotID, run.Size = staged.ID, staged.Size
+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
+ return run, saveErr
+ }
+ if progressErr := progress("local", scheduler.Succeeded, staged.ID); progressErr != nil {
+ return run, progressErr
+ }
} else {
- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
- }
- if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {
- err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
- }
- if err == nil {
- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})
- }
- return err
-}
-
-func (
- s *Service,
-) replicateBackupInternal(
- ctx context.Context,
- dockerClient *client.Client,
- prepared *preparedSystemBackupInternal,
- localRepository backup.Repository,
- stagedSnapshot backup.Snapshot,
-) error {
- run := prepared.run
- remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, run.S3DestinationID)
- if repoErr != nil {
- return repoErr
+ // S3-only runs forget the staged snapshot even when replication fails.
+ defer func() {
+ cleanupCtx := context.WithoutCancel(ctx)
+ if forgetErr := s.engine.ForgetSnapshots(cleanupCtx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {
+ slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshotId", staged.ID, "error", forgetErr)
+ }
+ }()
}
- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":remote", Status: scheduler.Running}); err != nil {
- return err
+ if !checkpoint.S3Enabled {
+ return run, nil
}
- remoteSnapshot, replicateErr := s.engine.Replicate(
- ctx,
- dockerClient,
- localRepository,
- stagedSnapshot.ID,
- remoteRepository,
- prepared.recoveryKey,
- "arcane-system-recovery",
- backup.RunSnapshotTag(
- run.ID,
- ),
- )
- if replicateErr != nil {
- return fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)
+ remote, err := s.remoteRepository(ctx, run.S3DestinationID)
+ if err != nil {
+ return run, err
}
- run.RemoteSnapshotID = remoteSnapshot.ID
- if run.Size == 0 {
- run.Size = remoteSnapshot.Size
+ if progressErr := progress("remote", scheduler.Running, ""); progressErr != nil {
+ return run, progressErr
}
- if err := s.store.SaveRun(ctx, run); err != nil {
- return err
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
+ if err != nil {
+ return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err)
}
- if err := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_destination",
- ID: run.ID + ":remote",
- Status: scheduler.Succeeded,
- Message: remoteSnapshot.ID,
- },
- ); err != nil {
- return err
+ run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)
+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
+ return run, saveErr
}
- return nil
+ return run, progress("remote", scheduler.Succeeded, snapshot.ID)
}
func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParams) ([]backuptypes.SystemBackupRun, pagination.Response, error) {
@@ -629,16 +525,11 @@ func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParams
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list system backups: %w", err)
}
- names := make(map[string]backuptypes.S3Destination)
- if s.s3Destinations != nil {
- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {
- names = available
- }
- }
- remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")
+ names := s.s3Destinations.DestinationsByID(ctx)
+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)
for i := range result {
result[i].S3DestinationName = names[result[i].S3DestinationID].Name
- result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, result[i].RemoteSnapshotID)
+ result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, "arcane-system-recovery", result[i].RemoteSnapshotID)
}
return result, response, nil
}
@@ -656,7 +547,7 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, reques
if run.Status != backuptypes.SystemBackupStatusSucceeded {
return nil, pagination.Response{}, errors.New("only successful system backups can be opened")
}
- key, err := s.recoveryKeyInternal(ctx, recoveryKey)
+ key, err := s.recoveryKey(ctx, recoveryKey)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -670,7 +561,7 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, reques
// RestoreBackupFiles restores selected project files into the current projects
// directory after creating a safety snapshot.
func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) error {
- lease, err := s.acquireRunInternal(ctx)
+ lease, err := s.acquireRun(ctx)
if err != nil {
return err
}
@@ -682,7 +573,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bac
if run.Status != backuptypes.SystemBackupStatusSucceeded {
return errors.New("only successful system backups can be restored")
}
- key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ key, err := s.recoveryKey(ctx, request.RecoveryKey)
if err != nil {
return err
}
@@ -690,17 +581,20 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bac
if err != nil {
return err
}
- return s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, func(
- ctx context.Context, safetyRequest backuptypes.CreateSystemBackupRequest,
- ) (*backuptypes.SystemBackupRun, error) {
- return s.createBackupInternal(ctx, backuptypes.SystemBackupTriggerSafety, safetyRequest)
- })
+ safetyBackup := func(ctx context.Context, safetyRequest backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
+ prepared, prepareErr := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerSafety, safetyRequest)
+ if prepareErr != nil {
+ return nil, prepareErr
+ }
+ return s.executeBackup(ctx, prepared)
+ }
+ return s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, safetyBackup)
}
func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) error {
// Deletes contend with create/restore/upload on the system run lease so a
// slow operation cannot resurrect or double-forget snapshots.
- lease, err := s.acquireRunInternal(ctx)
+ lease, err := s.acquireRun(ctx)
if err != nil {
return err
}
@@ -709,27 +603,27 @@ func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) erro
if err != nil {
return err
}
- return s.deleteRunsInternal(ctx, []*backuptypes.SystemBackupRun{run}, recoveryKey, true)
+ return s.deleteRuns(ctx, []*backuptypes.SystemBackupRun{run}, recoveryKey, true)
}
-// deleteRunsInternal forgets snapshots grouped by repository under the caller's run lease.
-func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backuptypes.SystemBackupRun, recoveryKey string, includeRemote bool) error {
+// deleteRuns forgets snapshots grouped by repository under the caller's run lease.
+func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.SystemBackupRun, recoveryKey string, includeRemote bool) error {
var localRuns []*backuptypes.SystemBackupRun
+ var localIDs []string
remoteGroups := make(map[string][]*backuptypes.SystemBackupRun)
- for _, run := range localRuns2 {
+ for _, run := range backupRuns {
if run.LocalSnapshotID != "" {
- localRuns = append(localRuns, run)
+ localRuns, localIDs = append(localRuns, run), append(localIDs, run.LocalSnapshotID)
}
if includeRemote && run.RemoteSnapshotID != "" {
remoteGroups[run.S3DestinationID] = append(remoteGroups[run.S3DestinationID], run)
}
}
- // A run without snapshots (a failed attempt) is just a row; deleting it
- // needs neither the key nor Rustic.
+ // A run without snapshots (a failed attempt) is just a row; deleting it needs neither the key nor Rustic.
key := ""
if len(localRuns) > 0 || len(remoteGroups) > 0 {
var err error
- if key, err = s.recoveryKeyInternal(ctx, recoveryKey); err != nil {
+ if key, err = s.recoveryKey(ctx, recoveryKey); err != nil {
return err
}
}
@@ -737,14 +631,27 @@ func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backupty
if err != nil {
return err
}
- deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, key, localRuns)
+ var deleteErr error
+ if len(localRuns) > 0 {
+ repository, repoErr := s.localRepository(ctx, dockerClient, false)
+ if repoErr == nil {
+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, localIDs)
+ }
+ if repoErr != nil {
+ deleteErr = fmt.Errorf("failed to delete local snapshots: %w", repoErr)
+ } else {
+ for _, run := range localRuns {
+ run.LocalSnapshotID = ""
+ }
+ }
+ }
for destinationID, group := range remoteGroups {
- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key, destinationID, group))
+ deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshots(ctx, dockerClient, key, destinationID, group))
}
- for _, run := range localRuns2 {
+ for _, run := range backupRuns {
if run.LocalSnapshotID == "" && run.RemoteSnapshotID == "" {
- if deleteBackupRunErr := s.store.DeleteRun(ctx, run.ID); deleteBackupRunErr != nil {
- deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete system backup record: %w", deleteBackupRunErr))
+ if deleteRunErr := s.store.DeleteRun(ctx, run.ID); deleteRunErr != nil {
+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete system backup record: %w", deleteRunErr))
}
continue
}
@@ -763,6 +670,34 @@ func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backupty
return deleteErr
}
+// forgetRemoteSnapshots forgets one S3 destination's snapshots and drops the runs' references to them.
+func (s *Service) forgetRemoteSnapshots(ctx context.Context, dockerClient *client.Client, key, destinationID string, group []*backuptypes.SystemBackupRun) error {
+ snapshotIDs := make([]string, len(group))
+ for index, run := range group {
+ snapshotIDs[index] = run.RemoteSnapshotID
+ }
+ repository, repoErr := s.remoteRepository(ctx, destinationID)
+ var observation backuptypes.RepositoryObservation
+ if repoErr == nil {
+ observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")
+ }
+ switch {
+ case repoErr != nil:
+ case !observation.Available:
+ // Nothing remains to forget; drop the references so the run can be removed.
+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))
+ default:
+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)
+ }
+ if repoErr != nil {
+ return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)
+ }
+ for _, run := range group {
+ run.RemoteSnapshotID, run.S3DestinationID = "", ""
+ }
+ return nil
+}
+
// PruneLocalRepository frees the data of deleted local system backups once Rustic's keep-delete window passes.
func (s *Service) PruneLocalRepository(ctx context.Context) error {
key, err := s.recoveryKeys.Get(ctx)
@@ -776,67 +711,19 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {
if err != nil {
return err
}
- repository, err := s.localRepositoryInternal(ctx, dockerClient, false)
+ repository, err := s.localRepository(ctx, dockerClient, false)
if err != nil {
return err
}
return s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, nil)
}
-func (s *Service) forgetLocalSnapshotsInternal(ctx context.Context, dockerClient *client.Client, key string, localRuns []*backuptypes.SystemBackupRun) error {
- if len(localRuns) == 0 {
- return nil
- }
- snapshotIDs := make([]string, len(localRuns))
- for index, run := range localRuns {
- snapshotIDs[index] = run.LocalSnapshotID
- }
- repository, repoErr := s.localRepositoryInternal(ctx, dockerClient, false)
- if repoErr == nil {
- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)
- }
- if repoErr != nil {
- return fmt.Errorf("failed to delete local snapshots: %w", repoErr)
- }
- for _, run := range localRuns {
- run.LocalSnapshotID = ""
- }
- return nil
-}
-
-func (s *Service) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, key, destinationID string, localRuns []*backuptypes.SystemBackupRun) error {
- snapshotIDs := make([]string, len(localRuns))
- for index, run := range localRuns {
- snapshotIDs[index] = run.RemoteSnapshotID
- }
- repository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)
- if repoErr == nil {
- var observation backuptypes.RepositoryObservation
- observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")
- switch {
- case repoErr != nil:
- case !observation.Available:
- // Nothing remains to forget; drop the references so the run can be removed.
- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))
- default:
- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)
- }
- }
- if repoErr != nil {
- return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)
- }
- for _, run := range localRuns {
- run.RemoteSnapshotID, run.S3DestinationID = "", ""
- }
- return nil
-}
-
func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {
- recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)
if err != nil {
return 0, err
}
- repository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)
+ repository, err := s.remoteRepository(ctx, request.S3DestinationID)
if err != nil {
return 0, err
}
@@ -852,28 +739,17 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes
if err != nil {
return 0, err
}
- known := make(map[string]struct{}, len(knownIDs))
- for _, id := range knownIDs {
- known[id] = struct{}{}
- }
created := 0
for _, snapshot := range listedSnapshots {
- if strings.TrimSpace(snapshot.ID) == "" {
- continue
- }
- if _, exists := known[snapshot.ID]; exists {
+ if strings.TrimSpace(snapshot.ID) == "" || slices.Contains(knownIDs, snapshot.ID) {
continue
}
- createdAt := snapshot.Time
- if createdAt.IsZero() {
- createdAt = time.Now().UTC()
- }
run := &backuptypes.SystemBackupRun{
- Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt, Status: backuptypes.SystemBackupStatusSucceeded,
+ ID: fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID), Size: snapshot.Summary.TotalBytesProcessed,
+ CreatedAt: kit.Ternary(snapshot.Time.IsZero(), time.Now().UTC(), snapshot.Time), Status: backuptypes.SystemBackupStatusSucceeded,
Trigger: backuptypes.SystemBackupTriggerManual, Destination: backuptypes.SystemBackupDestinationS3,
RemoteSnapshotID: snapshot.ID, S3DestinationID: request.S3DestinationID,
}
- run.ID = fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID)
if createDiscoveredBackupErr := s.store.CreateRun(ctx, run); createDiscoveredBackupErr != nil {
return created, fmt.Errorf("failed to save discovered system backup: %w", createDiscoveredBackupErr)
}
@@ -883,7 +759,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes
}
func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypes.UploadSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
- lease, err := s.acquireRunInternal(ctx)
+ lease, err := s.acquireRun(ctx)
if err != nil {
return nil, err
}
@@ -898,7 +774,7 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptyp
if run.RemoteSnapshotID != "" {
return nil, errors.New("system backup has already been uploaded")
}
- key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ key, err := s.recoveryKey(ctx, request.RecoveryKey)
if err != nil {
return nil, err
}
@@ -909,11 +785,11 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptyp
if err != nil {
return nil, err
}
- localRepository, err := s.localRepositoryInternal(ctx, dockerClient, true)
+ localRepository, err := s.localRepository(ctx, dockerClient, true)
if err != nil {
return nil, err
}
- remoteRepository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)
+ remoteRepository, err := s.remoteRepository(ctx, request.S3DestinationID)
if err != nil {
return nil, err
}
@@ -937,7 +813,7 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if run.Status != backuptypes.SystemBackupStatusSucceeded {
return errors.New("only successful system backups can be restored")
}
- key, err := s.recoveryKeyInternal(ctx, recoveryKey)
+ key, err := s.recoveryKey(ctx, recoveryKey)
if err != nil {
return err
}
@@ -959,17 +835,17 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if appDataMount == nil {
return errors.New("arcane system restore requires /app/data to be mounted")
}
- dataDirectory, err := s.dataDirectoryInternal()
+ databaseFile, err := s.databaseFile()
if err != nil {
return err
}
- projectsDirectory := s.projectsDirectoryInternal(ctx)
+ dataDirectory := filepath.Dir(databaseFile)
+ projectsDirectory := projects.ResolveConfiguredContainerDirectory(s.projectsSetting(ctx), "/app/data/projects")
plan, err := s.snapshots.PlanRestore(ctx, dockerClient, key, *run, current.Mounts, dataDirectory, projectsDirectory)
if err != nil {
return err
}
- // A local safety snapshot is created before the detached helper is allowed to
- // stop Arcane and replace its data.
+ // A local safety snapshot is created before the detached helper is allowed to stop Arcane and replace its data.
safetyBackup, err := s.CreateBackup(ctx, user, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{
Destination: backuptypes.SystemBackupDestinationLocal,
RecoveryKey: key,
@@ -981,15 +857,23 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if err != nil {
return err
}
+ // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.
+ rusticNetwork := ""
+ if current.NetworkSettings != nil {
+ names := slices.Sorted(maps.Keys(current.NetworkSettings.Networks))
+ if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {
+ rusticNetwork = names[index]
+ }
+ }
request := recovery.RestoreRequest{
BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: plan.SnapshotID, RecoveryKey: key,
LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,
- Size: run.Size, NetworkMode: rusticRestoreNetworkModeInternal(&current),
+ Size: run.Size, NetworkMode: rusticNetwork,
SafetyBackup: &recovery.SafetyBackup{
ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,
Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,
},
- ProjectsSetting: s.projectsSettingInternal(ctx), ProjectsIncluded: plan.ProjectsIncluded,
+ ProjectsSetting: s.projectsSetting(ctx), ProjectsIncluded: plan.ProjectsIncluded,
Stages: plan.Stages, RollbackStages: rollbackStages,
}
requestData, err := json.Marshal(request)
@@ -1025,14 +909,13 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if err != nil {
return fmt.Errorf("resolve Arcane executable: %w", err)
}
- helperExecutable, executableMount, err := recoveryHelperExecutableInternal(current.Mounts, executablePath)
- if err != nil {
- return fmt.Errorf("resolve recovery helper executable: %w", err)
- }
- mounts := append([]mount.Mount{}, runtimeMounts...)
- mounts = append(mounts, *appDataMount)
- if executableMount != nil {
- mounts = append(mounts, *executableMount)
+ helperExecutable, mounts := executablePath, append(slices.Clone(runtimeMounts), *appDataMount)
+ if executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath); executableMount != nil {
+ if executableMount.Type != mount.TypeBind {
+ return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")
+ }
+ executableMount.ReadOnly = true
+ helperExecutable, mounts = systemRecoveryHelperPath, append(mounts, *executableMount)
}
hostConfig := &container.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}
if current.HostConfig != nil {
@@ -1059,22 +942,6 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
return nil
}
-// rusticRestoreNetworkModeInternal picks a named network Arcane is attached to
-// so the detached restore's Rustic container can still resolve network-local
-// S3 endpoints once Arcane is stopped. The default bridge carries no DNS, so
-// it (and none) fall back to the default network.
-func rusticRestoreNetworkModeInternal(current *container.InspectResponse) string {
- if current == nil || current.NetworkSettings == nil {
- return ""
- }
- for _, name := range slices.Sorted(maps.Keys(current.NetworkSettings.Networks)) {
- if name != "bridge" && name != "none" {
- return name
- }
- }
- return ""
-}
-
func (s *Service) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {
recoveryKey, err := backup.GenerateRecoveryKey()
if err != nil {
@@ -1087,15 +954,13 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*back
if err := backup.ValidateRecoveryKey(recoveryKey); err != nil {
return nil, err
}
- // The repositories are encrypted with the key that initialized them and a
- // different key cannot open them, so a rotation would silently break every
- // scheduled backup and keyless restore. Refuse it while backups exist.
+ // Repositories only open with the key that initialized them, so refuse a rotation while backups exist.
configured, err := s.recoveryKeys.Configured(ctx)
if err != nil {
return nil, err
}
if configured {
- if current, keyErr := s.recoveryKeyInternal(ctx, ""); keyErr == nil && current != recoveryKey {
+ if current, keyErr := s.recoveryKeys.Get(ctx); keyErr == nil && current != recoveryKey {
localRuns, countLocalBackupsErr := s.store.CountRuns(ctx)
if countLocalBackupsErr != nil {
return nil, countLocalBackupsErr
@@ -1135,12 +1000,7 @@ func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPol
result := &backuptypes.SystemBackupPolicyCollection{
Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,
}
- destinations := make(map[string]backuptypes.S3Destination)
- if s.s3Destinations != nil {
- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {
- destinations = available
- }
- }
+ destinations := s.s3Destinations.DestinationsByID(ctx)
for i := range policies {
lastRun, runErr := s.store.LatestPolicyRun(ctx, policies[i].ID)
if runErr != nil {
@@ -1178,7 +1038,7 @@ func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.Upda
policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationID
return nil
}
- if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicyInternal); runErr != nil {
+ if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicy); runErr != nil {
return nil, runErr
}
return s.GetPolicies(ctx)
@@ -1190,7 +1050,7 @@ func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.D
return s.jobs.SetScheduler(ctx, dynamicScheduler, admissionGate)
}
-func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {
+func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (scheduler.Outcome, error) {
policy, loadErr := s.store.Policy(ctx, policyID)
if loadErr != nil {
return scheduler.Outcome{}, loadErr
@@ -1198,64 +1058,64 @@ func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID strin
if policy == nil || !policy.Enabled {
return scheduler.Outcome{Status: scheduler.Skipped}, nil
}
- if previous, ok := jobcontext.Run(ctx); ok {
- outcome := jobcontext.ConfirmedTarget(previous, policy.ID)
- if outcome.Status == scheduler.Succeeded {
- if policy.RetentionCount > 0 {
- if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); err != nil {
- outcome.Status = scheduler.Partial
- return outcome, err
- }
- }
- return outcome, nil
- }
+ var remoteErr error
+ if policy.S3Enabled {
+ remoteErr = s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)
}
- remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, policy)
- if checkErr != nil {
- return scheduler.Outcome{}, checkErr
+ remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {
+ return s.store.DisablePolicyRemote(ctx, policy, field)
+ })
+ if disableErr != nil {
+ return scheduler.Outcome{}, disableErr
+ }
+ if remoteDisabled {
+ s.rescheduleSystemBackupPolicy(ctx, policy)
}
if remoteDisabled && !policy.LocalEnabled {
return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
}
- var run *backuptypes.SystemBackupRun
- activityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
- EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",
- ResourceID: policy.ID, ResourceName: "Arcane", User: &usertypes.SystemUser,
- Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",
- SuccessMessage: "Scheduled Arcane system backup created successfully",
- Metadata: database.JSON{
- "action": "scheduled_system_backup", "policyId": policy.ID, "schedule": policy.Schedule,
- "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,
- "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,
- },
- }, func(activityCtx context.Context) error {
- var backupErr error
- run, backupErr = s.CreateBackup(activityCtx, usertypes.SystemUser, backuptypes.SystemBackupTriggerScheduled,
- backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})
- return backupErr
- })
+ run, runErr := s.CreateBackup(ctx, usertypes.SystemUser, backuptypes.SystemBackupTriggerScheduled, backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})
if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {
slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)
- return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: another system backup is running"}, nil
}
if runErr != nil {
slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)
- return scheduler.Outcome{ActivityID: activityID}, runErr
+ return scheduler.Outcome{}, runErr
}
if policy.RetentionCount > 0 {
- if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {
+ if retentionErr := s.applyRetention(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {
slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)
- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr
+ return scheduler.Outcome{Status: scheduler.Partial, Message: "Backup completed but retention failed"}, retentionErr
}
}
slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)
if remoteDisabled {
- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil
+ return scheduler.Outcome{Status: scheduler.Partial, Message: backup.RemoteDisabledMessage}, nil
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Scheduled Arcane system backup created successfully"}, nil
+}
+
+// resumeScheduledBackup finishes the backup an interrupted scheduled run recorded, then applies retention.
+func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.Run, policyID string) (scheduler.Outcome, error) {
+ outcome, err := s.reconcileBackup(ctx, previous, policyID, "")
+ if err != nil || outcome.Status != scheduler.Succeeded {
+ return outcome, err
+ }
+ current, err := s.store.Policy(ctx, policyID)
+ if err != nil {
+ return outcome, err
+ }
+ if current != nil && current.RetentionCount > 0 {
+ if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {
+ outcome.Status = scheduler.Partial
+ return outcome, applyRetentionErr
+ }
}
- return scheduler.Outcome{Status: scheduler.Succeeded, ActivityID: activityID}, nil
+ return outcome, nil
}
-func (s *Service) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {
+func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {
if policy == nil {
return
}
@@ -1264,35 +1124,30 @@ func (s *Service) rescheduleSystemBackupPolicyInternal(ctx context.Context, poli
return
}
policyID := policy.ID
- s.jobs.Register(ctx, policyID,
- func(ctx context.Context) string {
+ s.jobs.Add(ctx, &flow.Job{
+ Engine: s.flow,
+ Workflow: s.policyWorkflow,
+ JobName: s.jobs.JobName(policyID),
+ Payload: policyID,
+ Activity: activitylib.StartRequest{
+ ResourceID: new(policyID),
+ Metadata: database.JSON{
+ "action": "scheduled_system_backup", "policyId": policyID, "schedule": policy.Schedule,
+ "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,
+ "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,
+ },
+ },
+ ScheduleFn: func(ctx context.Context) string {
current, err := s.store.Policy(ctx, policyID)
if err != nil || current == nil {
return defaultSystemBackupSchedule
}
return current.Schedule
},
- func(ctx context.Context) (scheduler.Outcome, error) {
- return s.runScheduledBackupInternal(ctx, policyID)
+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
+ return s.resumeScheduledBackup(ctx, previous, policyID)
},
- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
- outcome, err := s.reconcileBackupInternal(ctx, previous, policyID)
- if err != nil || outcome.Status != scheduler.Succeeded {
- return outcome, err
- }
- current, err := s.store.Policy(ctx, policyID)
- if err != nil {
- return outcome, err
- }
- if current != nil && current.RetentionCount > 0 {
- if applyRetentionErr := s.applyRetentionInternal(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {
- outcome.Status = scheduler.Partial
- return outcome, applyRetentionErr
- }
- }
- return outcome, nil
- },
- )
+ })
}
func (s *Service) RegisterBackupJobOnStartup(ctx context.Context) {
@@ -1302,52 +1157,31 @@ func (s *Service) RegisterBackupJobOnStartup(ctx context.Context) {
return
}
for i := range policies {
- s.rescheduleSystemBackupPolicyInternal(ctx, &policies[i])
+ s.rescheduleSystemBackupPolicy(ctx, &policies[i])
}
volumePolicyCount := s.volumes.RegisterJobsOnStartup(ctx)
slog.InfoContext(ctx, "Registered backup schedules", "systemPolicies", len(policies), "volumePolicies", volumePolicyCount)
}
-func (s *Service) applyRetentionInternal(ctx context.Context, policyID string, keep int, includeRemote bool) error {
+func (s *Service) applyRetention(ctx context.Context, policyID string, keep int, includeRemote bool) error {
expired, err := s.store.ExpiredRunIDs(ctx, policyID, keep)
if err != nil || len(expired) == 0 {
return err
}
- lease, err := s.acquireRunInternal(ctx)
+ lease, err := s.acquireRun(ctx)
if err != nil {
return err
}
defer lease.Release(ctx)
- localRuns, err := s.store.RunsByID(ctx, expired)
+ expiredRuns, err := s.store.RunsByID(ctx, expired)
if err != nil {
return err
}
- return s.deleteRunsInternal(ctx, localRuns, "", includeRemote)
+ return s.deleteRuns(ctx, expiredRuns, "", includeRemote)
}
-func (s *Service) disableMissingS3Internal(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error) {
- if !policy.S3Enabled {
- return false, nil
- }
- err := s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)
- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
- return false, nil
- }
- disabled, err := s.store.DisablePolicyRemote(ctx, policy)
- if err != nil || !disabled {
- return false, err
- }
- if policy.LocalEnabled {
- policy.S3Enabled = false
- } else {
- policy.Enabled = false
- }
- s.rescheduleSystemBackupPolicyInternal(ctx, policy)
- return true, nil
-}
-
-// projectsSettingInternal returns the effective projectsDirectory setting.
-func (s *Service) projectsSettingInternal(ctx context.Context) string {
+// projectsSetting returns the effective projectsDirectory setting.
+func (s *Service) projectsSetting(ctx context.Context) string {
fallback := ""
if s.config != nil {
fallback = s.config.ProjectsDirectory
@@ -1358,178 +1192,181 @@ func (s *Service) projectsSettingInternal(ctx context.Context) string {
return s.settingsService.GetStringSetting(ctx, "projectsDirectory", fallback)
}
-// projectsDirectoryInternal resolves the configured projects directory to the
-// container path Arcane reads, dropping any host-path mapping suffix.
-func (s *Service) projectsDirectoryInternal(ctx context.Context) string {
- return projects.ResolveConfiguredContainerDirectory(s.projectsSettingInternal(ctx), "/app/data/projects")
-}
-
-// StartBackup prepares a manual backup before submitting its snapshot work.
+// StartBackup prepares a manual backup, then submits its workflow.
func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
- lease, err := s.acquireRunInternal(ctx)
+ requester := backup.NewRequester(ctx, user, "0", authz.PermSystemBackupsManage)
+ if err := s.engine.Authorize(ctx, requester); err != nil {
+ return nil, err
+ }
+ lease, err := s.acquireRun(ctx)
if err != nil {
return nil, err
}
- prepared, err := s.prepareBackupInternal(ctx, backuptypes.SystemBackupTriggerManual, request)
+ prepared, err := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerManual, request)
if err != nil {
lease.Release(ctx)
return nil, err
}
- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "arcane", "Arcane", &user,
- "Creating system backup", "Creating Arcane system backup", database.JSON{
- "action": "create_system_backup",
- "backupId": prepared.run.ID,
- "destination": prepared.run.Destination,
- "s3DestinationId": prepared.run.S3DestinationID,
- "policyId": prepared.run.PolicyID,
- }, false)
- finish := func(runErr error) {
- defer lease.Release(ctx)
- if runErr != nil {
- if saveErr := s.store.FailRun(context.WithoutCancel(workCtx), prepared.run.ID, runErr.Error()); saveErr != nil {
- runErr = errors.Join(runErr, fmt.Errorf("save backup failure: %w", saveErr))
- }
+ run := prepared.run
+ release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, run.ID, lease)
+ // An entered key is held encrypted, outside the workflow payload, so a restart can still resume the backup.
+ if request.RecoveryKey != "" {
+ if holdErr := s.recoveryKeys.HoldRunKey(ctx, run.ID, prepared.recoveryKey); holdErr != nil {
+ release(ctx)
+ return nil, errors.Join(holdErr, s.store.FailRun(context.WithoutCancel(ctx), run.ID, holdErr.Error()))
}
- activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Arcane system backup created successfully", runErr)
}
- if activityID == "" {
- err = errors.New("failed to create system backup activity")
- finish(err)
- return nil, err
+ input := manualBackupInput{Checkpoint: prepared.checkpoint, Requester: requester}
+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{
+ ResourceID: new("arcane"), StartedBy: &user, Step: "Creating system backup", LatestMessage: "Creating Arcane system backup",
+ Metadata: database.JSON{"action": "create_system_backup", "backupId": run.ID, "destination": run.Destination, "s3DestinationId": run.S3DestinationID, "policyId": run.PolicyID},
+ }, backup.AcceptedTarget(run.ID))
+ if err != nil {
+ release(ctx)
+ return nil, errors.Join(err, s.recoveryKeys.ReleaseRunKey(context.WithoutCancel(ctx), run.ID), s.store.FailRun(context.WithoutCancel(ctx), run.ID, err.Error()))
}
- dto := *prepared.run
+ // A backup canceled before its task runs never takes the parked lease or uses the held key.
+ go func() {
+ background := context.WithoutCancel(ctx)
+ s.watchBackup(background, run.ID, activityID)
+ release(background)
+ }()
+ dto := *run
dto.ActivityID = activityID
- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
- encryptedKey, keyErr := crypto.Encrypt(prepared.recoveryKey)
- if keyErr != nil {
- finish(keyErr)
- return nil, keyErr
- }
- payload, err := json.Marshal(
- manualSystemBackupInternal{
- Checkpoint: systemBackupRecoveryInternal{
- BackupID: prepared.run.ID,
- LocalEnabled: prepared.localEnabled,
- S3Enabled: prepared.s3Enabled,
- },
- EncryptedKey: encryptedKey,
- ActivityID: activityID,
- UserID: user.ID,
- },
- )
- if err == nil {
- err = s.engine.SubmitDurableRun(
- workCtx,
- backuptypes.DurableRunCommand{
- Kind: "system",
- RunID: prepared.run.ID,
- ActivityID: activityID,
- Payload: payload,
- UserID: user.ID,
- EnvironmentID: "0",
- Permission: authz.PermSystemBackupsManage,
- RequestedWithKey: keyID,
- },
- lease,
- )
- }
+ return &dto, nil
+}
+
+// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.
+func (s *Service) WatchAcceptedBackups(ctx context.Context) error {
+ standalone, err := s.flow.Standalone(ctx)
if err != nil {
- finish(err)
- return nil, err
+ return err
}
-
- return &dto, nil
+ for _, run := range standalone {
+ if backupID := backup.AcceptedBackupID(run); backupID != "" && run.JobID == s.backupWorkflow.Francis().Name() {
+ go s.watchBackup(context.WithoutCancel(ctx), backupID, run.ID)
+ }
+ }
+ return nil
}
-func (s *Service) reconcileBackupInternal(ctx context.Context, previous scheduler.Run, policyID string, suppliedKeys ...string) (scheduler.Outcome, error) {
- outcome := jobcontext.ConfirmedTarget(previous, policyID)
+// watchBackup releases the run's held key once its workflow ends, and fails the run's row when the workflow
+// ended without the task completing it, such as a cancel before the task started.
+func (s *Service) watchBackup(ctx context.Context, runID, instanceID string) {
+ outcome, err := s.flow.Wait(ctx, s.backupWorkflow, instanceID)
+ if err != nil {
+ return
+ }
+ if releaseErr := s.recoveryKeys.ReleaseRunKey(ctx, runID); releaseErr != nil {
+ slog.WarnContext(ctx, "Failed to release system backup recovery key", "runId", runID, "error", releaseErr)
+ }
if outcome.Status == scheduler.Succeeded {
+ return
+ }
+ if current, loadErr := s.store.Run(ctx, runID); loadErr == nil && current.Status == backuptypes.SystemBackupStatusRunning {
+ if failErr := s.store.FailRun(ctx, runID, cmp.Or(outcome.Message, "Backup did not run")); failErr != nil {
+ slog.WarnContext(ctx, "Failed to close a system backup whose workflow ended early", "runId", runID, "error", failErr)
+ }
+ }
+}
+
+// reconcileBackup finishes the backup checkpointed under targetID from the snapshots its interrupted run left behind.
+func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, targetID, suppliedKey string) (scheduler.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, targetID)
+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == targetID && len(target.RecoveryData) > 0
+ })
+ if outcome.Status == scheduler.Succeeded || index < 0 {
return outcome, nil
}
- for _, target := range previous.Outcome.Targets {
- if target.ID != policyID || len(target.RecoveryData) == 0 {
- continue
+ target := previous.Outcome.Targets[index]
+ var checkpoint systemBackupRecovery
+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return outcome, err
+ }
+ confirm := func() (scheduler.Outcome, error) {
+ target.Status = scheduler.Succeeded
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return outcome, progressErr
}
- var checkpoint systemBackupRecoveryInternal
- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
+ }
+ run, err := s.store.Run(ctx, checkpoint.BackupID)
+ if err != nil {
+ return outcome, err
+ }
+ if run.Status == backuptypes.SystemBackupStatusSucceeded {
+ return confirm()
+ }
+ lease, admitted, err := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, "")
+ if err != nil || !admitted {
+ return outcome, err
+ }
+ defer lease.Release(ctx)
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return outcome, err
+ }
+ key, err := s.recoveryKey(ctx, suppliedKey)
+ if err != nil {
+ return outcome, err
+ }
+ staged, local, remote, err := s.observeBackupSnapshots(ctx, dockerClient, previous, run, checkpoint, key)
+ if err != nil {
+ return outcome, err
+ }
+ if checkpoint.LocalEnabled && staged.ID == "" && run.RemoteSnapshotID != "" {
+ if staged, err = s.engine.Replicate(ctx, dockerClient, remote, run.RemoteSnapshotID, local, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID)); err != nil {
return outcome, err
}
- run, loadRunErr := s.store.Run(ctx, checkpoint.BackupID)
- if loadRunErr != nil {
- return outcome, loadRunErr
- }
- if run.Status != backuptypes.SystemBackupStatusSucceeded {
- lease, admitted, err := s.engine.AcquireDurableRun(ctx, previous.ID, backup.SystemAdmissionScope, systemAdmissionID)
- if err != nil {
- return outcome, err
- }
- if !admitted {
- return outcome, nil
- }
- defer lease.Release(ctx) //nolint:gocritic // The matching recovery target always returns before the loop advances.
- key := ""
- if len(suppliedKeys) > 0 {
- key = suppliedKeys[0]
- }
- if resumeBackupErr := s.resumeBackupInternal(ctx, previous, run, checkpoint, key); resumeBackupErr != nil {
- return outcome, resumeBackupErr
- }
- }
- target.Status = scheduler.Succeeded
- if err := jobcontext.Progress(ctx, target); err != nil {
+ }
+ if staged.ID == "" && run.RemoteSnapshotID == "" {
+ // Nothing survived the interruption, so the backup runs again.
+ if _, err = s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: checkpoint}); err != nil {
return outcome, err
}
- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
+ return confirm()
}
- return outcome, nil
-}
-
-func systemBackupStageEvidenceInternal(previous scheduler.Run, run *backuptypes.SystemBackupRun) (string, bool) {
- stageID, attempted := run.LocalSnapshotID, false
- for _, evidence := range previous.Outcome.Targets {
- if evidence.ID != run.ID+":stage" {
- continue
- }
- attempted = true
- if stageID == "" && evidence.Status == scheduler.Succeeded {
- stageID = evidence.Message
+ if checkpoint.LocalEnabled {
+ run.LocalSnapshotID, run.Size = staged.ID, staged.Size
+ }
+ // A missing remote snapshot implies a staged one; the rerun above handled neither surviving.
+ if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {
+ snapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
+ if replicateErr != nil {
+ return outcome, replicateErr
}
+ run.RemoteSnapshotID = snapshot.ID
}
- return stageID, attempted
-}
-
-func systemBackupRemoteAttemptedInternal(previous scheduler.Run, runID string) bool {
- for _, evidence := range previous.Outcome.Targets {
- if evidence.ID == runID+":remote" {
- return true
+ if !checkpoint.LocalEnabled && staged.ID != "" {
+ if forgetErr := s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {
+ return outcome, forgetErr
}
}
- return false
+ run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {
+ return outcome, saveErr
+ }
+ return confirm()
}
-func (
- s *Service,
-) observeBackupSnapshotsInternal(
- ctx context.Context,
- dockerClient *client.Client,
- previous scheduler.Run,
- run *backuptypes.SystemBackupRun,
- checkpoint systemBackupRecoveryInternal,
- key string,
-) (
- backup.Snapshot,
- backup.Repository,
- backup.Repository,
- error,
-) {
+// observeBackupSnapshots finds the snapshots an interrupted run already wrote and records any remote one on run.
+func (s *Service) observeBackupSnapshots(
+ ctx context.Context, dockerClient *client.Client, previous scheduler.Run, run *backuptypes.SystemBackupRun, checkpoint systemBackupRecovery, key string,
+) (backup.Snapshot, backup.Repository, backup.Repository, error) {
var staged backup.Snapshot
var remote backup.Repository
- local, err := s.localRepositoryInternal(ctx, dockerClient, false)
+ local, err := s.localRepository(ctx, dockerClient, false)
if err != nil && checkpoint.LocalEnabled {
return staged, local, remote, err
}
- stageID, stageAttempted := systemBackupStageEvidenceInternal(previous, run)
- if err == nil && (stageAttempted || stageID != "") {
+ targets := previous.Outcome.Targets
+ stageIndex := slices.IndexFunc(targets, func(target scheduler.TargetOutcome) bool { return target.ID == run.ID+":stage" })
+ stageID := run.LocalSnapshotID
+ if stageIndex >= 0 && stageID == "" && targets[stageIndex].Status == scheduler.Succeeded {
+ stageID = targets[stageIndex].Message
+ }
+ if err == nil && (stageIndex >= 0 || stageID != "") {
snapshot, found, findErr := s.engine.FindRunSnapshot(ctx, dockerClient, local, key, run.ID, stageID)
if findErr == nil && found {
staged = snapshot
@@ -1541,163 +1378,96 @@ func (
if !checkpoint.S3Enabled {
return staged, local, remote, nil
}
- remote, err = s.remoteRepositoryInternal(ctx, run.S3DestinationID)
+ remote, err = s.remoteRepository(ctx, run.S3DestinationID)
if err != nil {
return staged, local, remote, err
}
- attempted := (stageAttempted && staged.ID == "") || systemBackupRemoteAttemptedInternal(previous, run.ID)
- if attempted || run.RemoteSnapshotID != "" {
- snapshot, found, findRunSnapshotErr := s.engine.FindRunSnapshot(ctx, dockerClient, remote, key, run.ID, run.RemoteSnapshotID)
- if findRunSnapshotErr != nil {
- return staged, local, remote, findRunSnapshotErr
+ remoteAttempted := slices.ContainsFunc(targets, func(target scheduler.TargetOutcome) bool { return target.ID == run.ID+":remote" })
+ if (stageIndex >= 0 && staged.ID == "") || remoteAttempted || run.RemoteSnapshotID != "" {
+ snapshot, found, findErr := s.engine.FindRunSnapshot(ctx, dockerClient, remote, key, run.ID, run.RemoteSnapshotID)
+ if findErr != nil {
+ return staged, local, remote, findErr
}
if found {
- run.RemoteSnapshotID = snapshot.ID
- if run.Size == 0 {
- run.Size = snapshot.Size
- }
+ run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)
}
}
return staged, local, remote, nil
}
-func (
- s *Service,
-) finishRecoveredDestinationsInternal(
- ctx context.Context,
- dockerClient *client.Client,
- run *backuptypes.SystemBackupRun,
- checkpoint systemBackupRecoveryInternal,
- key string,
- staged backup.Snapshot,
- local, remote backup.Repository,
-) error {
- if checkpoint.LocalEnabled && staged.ID == "" && run.RemoteSnapshotID != "" {
- var err error
- staged, err = s.engine.Replicate(ctx, dockerClient, remote, run.RemoteSnapshotID, local, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
- if err != nil {
- return err
- }
- }
- if staged.ID == "" && run.RemoteSnapshotID == "" {
- _, err := s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: run, recoveryKey: key, localEnabled: checkpoint.LocalEnabled, s3Enabled: checkpoint.S3Enabled})
- return err
- }
- if checkpoint.LocalEnabled {
- run.LocalSnapshotID, run.Size = staged.ID, staged.Size
+// runManualBackup is the system-backup step. A delivery that finds the checkpoint it recorded
+// resumes that backup; otherwise it takes over the admission StartBackup won and runs it.
+func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, err error) {
+ var input manualBackupInput
+ if decodeErr := t.Payload(&input); decodeErr != nil {
+ return nil, decodeErr
}
- if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {
- if staged.ID == "" {
- return errors.New("System backup has no snapshot to replicate") //nolint:staticcheck // Preserve the existing error message.
+ runID := input.Checkpoint.BackupID
+ defer func() {
+ // Shutdown leaves the run, and any key held for it, to the redelivered task.
+ if ctx.Err() != nil {
+ return
}
- snapshot, err := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
if err != nil {
- return err
+ err = errors.Join(err, s.store.FailRun(ctx, runID, err.Error()))
}
- run.RemoteSnapshotID = snapshot.ID
+ err = errors.Join(err, s.recoveryKeys.ReleaseRunKey(ctx, runID))
+ }()
+ previous, _ := jobcontext.Run(ctx)
+ resuming := slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == runID && len(target.RecoveryData) > 0
+ })
+ if !resuming {
+ lease, admitted, acquireErr := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, runID)
+ if acquireErr != nil {
+ return nil, acquireErr
+ }
+ if !admitted {
+ return nil, ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release(ctx)
}
- if !checkpoint.LocalEnabled && staged.ID != "" {
- return s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID})
+ if authorizeErr := s.engine.Authorize(ctx, input.Requester); authorizeErr != nil {
+ return nil, authorizeErr
}
- return nil
-}
-
-func (s *Service) resumeBackupInternal(ctx context.Context, previous scheduler.Run, run *backuptypes.SystemBackupRun, checkpoint systemBackupRecoveryInternal, suppliedKey string) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ succeeded := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Arcane system backup created successfully"}
+ run, err := s.store.Run(ctx, runID)
if err != nil {
- return err
+ return nil, err
}
- key, err := s.recoveryKeyInternal(ctx, suppliedKey)
- if err != nil {
- return err
+ if run.Status == backuptypes.SystemBackupStatusSucceeded {
+ return succeeded, nil
}
- staged, local, remote, err := s.observeBackupSnapshotsInternal(ctx, dockerClient, previous, run, checkpoint, key)
+ supplied, err := s.recoveryKeys.RunKey(ctx, runID)
if err != nil {
- return err
- }
- if finishRecoveredDestinationsErr := s.finishRecoveredDestinationsInternal(ctx, dockerClient, run, checkpoint, key, staged, local, remote); finishRecoveredDestinationsErr != nil {
- return finishRecoveredDestinationsErr
- }
- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""
- return s.store.SaveRun(ctx, run)
-}
-
-type manualSystemBackupInternal struct {
- Checkpoint systemBackupRecoveryInternal `json:"checkpoint"`
- EncryptedKey string `json:"encryptedKey"`
- ActivityID string `json:"activityId"`
- UserID string `json:"userId"`
-}
-
-func (s *Service) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
- var command manualSystemBackupInternal
- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {
- return decodeErr
- }
- defer func() {
- if ctx.Err() == nil {
- if err != nil {
- saveErr := s.store.FailRun(ctx, command.Checkpoint.BackupID, err.Error())
- err = errors.Join(err, saveErr)
- }
- activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", err)
- }
- }()
- run, loadBackupErr := s.store.Run(ctx, command.Checkpoint.BackupID)
- if loadBackupErr != nil {
- return loadBackupErr
- }
- if run.Status == backuptypes.SystemBackupStatusSucceeded {
- return nil
+ return nil, err
}
- key, err := crypto.Decrypt(command.EncryptedKey)
+ key, err := s.recoveryKey(ctx, supplied)
if err != nil {
- return err
+ return nil, err
}
- if interrupted {
- previous, _ := jobcontext.Run(ctx)
- outcome, recoveryErr := s.reconcileBackupInternal(ctx, previous, runID, key)
- if recoveryErr != nil {
- return recoveryErr
+ if resuming {
+ outcome, reconcileErr := s.reconcileBackup(ctx, previous, runID, key)
+ if reconcileErr != nil {
+ return nil, reconcileErr
}
if outcome.Status != scheduler.Succeeded {
- return errors.New(outcome.Message)
+ return nil, errors.New(outcome.Message)
}
- return nil
- }
- lease, err := s.acquireDurableRunInternal(ctx, runID)
- if err != nil {
- return err
+ return succeeded, nil
}
- defer lease.Release(ctx)
- checkpoint, err := json.Marshal(command.Checkpoint)
+ checkpoint, err := json.Marshal(input.Checkpoint)
if err != nil {
- return err
+ return nil, err
}
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "system_backup",
- ID: runID,
- Status: scheduler.Running,
- RecoveryData: checkpoint,
- ActivityID: command.ActivityID,
- },
- ); progressErr != nil {
- return progressErr
+ target := scheduler.TargetOutcome{ResourceType: "system_backup", ID: runID, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return nil, progressErr
}
- _, err = s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: run, recoveryKey: key, localEnabled: command.Checkpoint.LocalEnabled, s3Enabled: command.Checkpoint.S3Enabled})
- return err
-}
-
-func (s *Service) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {
- var command manualSystemBackupInternal
- if err := json.Unmarshal(payload, &command); err != nil {
- return err
+ if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: input.Checkpoint}); executeErr != nil {
+ return nil, executeErr
}
- err := s.store.FailRun(ctx, command.Checkpoint.BackupID, runErr.Error())
- activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", runErr)
- return err
+ return succeeded, nil
}
// ListBackupHistory returns a unified, server-paginated view of local backup records.
@@ -1706,31 +1476,25 @@ func (s *Service) ListBackupHistory(ctx context.Context, params pagination.Query
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("list backup history: %w", err)
}
- decorateHistoryDestinationsInternal(ctx, s.s3Destinations, history)
- systemAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")
- volumeRoot := ""
+ destinations := s.s3Destinations.DestinationsByID(ctx)
+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)
+ instanceID := ""
if s.settingsService != nil {
- volumeRoot = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value
+ instanceID = s.settingsService.GetSettingsConfig().InstanceID.Value
}
- volumeAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, volumeRoot)
for i := range history {
- check := kit.Ternary(history[i].ResourceType == "volume", volumeAvailable, systemAvailable)
- history[i].RemoteAvailable = check(history[i].S3DestinationID, history[i].RemoteSnapshotID)
+ history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name
+ // Volume backups discovered from another instance live under that instance's root.
+ root := "arcane-system-recovery"
+ if history[i].ResourceType == "volume" {
+ root = ""
+ if owner := cmp.Or(history[i].RemoteInstanceID, instanceID); owner != "" {
+ root = path.Join(backup.VolumeRoot, owner)
+ }
+ }
+ history[i].RemoteAvailable = remoteAvailable(history[i].S3DestinationID, root, history[i].RemoteSnapshotID)
}
page.GrandTotalItems = page.TotalItems
return history, page, nil
}
-
-func decorateHistoryDestinationsInternal(ctx context.Context, service *s3.S3DestinationService, history []backuptypes.HistoryEntry) {
- if service == nil {
- return
- }
- destinations, err := service.ListS3DestinationsByID(ctx)
- if err != nil {
- return
- }
- for i := range history {
- history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name
- }
-}
diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.go
index 8f772caeb..3657612a3 100644
--- a/backend/internal/system/children/backup/service_test.go
+++ b/backend/internal/system/children/backup/service_test.go
@@ -1,38 +1,56 @@
package backup
import (
+ "context"
"testing"
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/libtnb/sqlite"
- "github.com/moby/moby/api/types/container"
- "github.com/moby/moby/api/types/mount"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
)
-func TestRecoveryHelperExecutableInternal(t *testing.T) {
- path, executableMount, err := recoveryHelperExecutableInternal(nil, "/app/arcane")
- require.NoError(t, err)
- require.Equal(t, "/app/arcane", path)
- require.Nil(t, executableMount)
+func TestProjectsSettingFallsBackToConfig(t *testing.T) {
+ service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}
+ require.Equal(t, "/app/data/projects:/host/projects", service.projectsSetting(t.Context()))
+}
- path, executableMount, err = recoveryHelperExecutableInternal([]container.MountPoint{{
- Type: mount.TypeBind, Source: "/workspace/backend", Destination: "/app/backend", RW: true,
- }}, "/app/backend/.bin/arcane")
+func TestHistoryDestinationDecoration(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.Equal(t, systemRecoveryHelperPath, path)
- require.Equal(t, mount.TypeBind, executableMount.Type)
- require.Equal(t, "/workspace/backend/.bin/arcane", executableMount.Source)
- require.Equal(t, systemRecoveryHelperPath, executableMount.Target)
- require.True(t, executableMount.ReadOnly)
+ require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))
+ destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}
+ require.NoError(t, db.Create(&destination).Error)
+ service := &Service{
+ s3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil),
+ store: Store{ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {
+ return []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}, pagination.Response{}, nil
+ }},
+ }
+ history, _, err := service.ListBackupHistory(t.Context(), pagination.QueryParams{})
+ require.NoError(t, err)
+ require.Equal(t, "Offsite", history[0].S3DestinationName)
+ require.Empty(t, history[1].S3DestinationName)
+ require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))
+ history, _, err = service.ListBackupHistory(t.Context(), pagination.QueryParams{})
+ require.NoError(t, err)
+ require.Empty(t, history[0].S3DestinationName)
+}
+
+func TestBackupWorkflowDefinitions(t *testing.T) {
+ harness := flowtest.New(t, nil)
+ require.NoError(t, NewService(Dependencies{}).RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
}
-func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {
+func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {
cfg := &config.Config{
JWTSecret: "jwt-secret",
EncryptionKey: "encryption-secret",
@@ -40,34 +58,10 @@ func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {
OidcClientSecret: "oidc-secret",
FilePerm: 0o640,
}
- environment := (&Service{config: cfg}).recoveryEnvironmentInternal(t.Context())
+ environment := (&Service{config: cfg}).recoveryEnvironment(t.Context())
require.Equal(t, "jwt-secret", environment["JWT_SECRET"])
require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])
require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])
require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])
require.Equal(t, "0640", environment["FILE_PERM"])
}
-
-func TestProjectsDirectoryInternalUsesContainerPathOfMapping(t *testing.T) {
- service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}
- require.Equal(t, "/app/data/projects", service.projectsDirectoryInternal(t.Context()))
- require.Equal(t, "/app/data/projects:/host/projects", service.projectsSettingInternal(t.Context()))
-}
-
-func TestHistoryDestinationDecorationInternal(t *testing.T) {
- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
- require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))
- destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}
- require.NoError(t, db.Create(&destination).Error)
- service := s3.NewS3DestinationService(&database.DB{DB: db}, nil)
- history := []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}
- decorateHistoryDestinationsInternal(t.Context(), service, history)
- require.Equal(t, "Offsite", history[0].S3DestinationName)
- require.Empty(t, history[1].S3DestinationName)
- require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))
- decorateHistoryDestinationsInternal(t.Context(), service, history)
- require.Equal(t, "Offsite", history[0].S3DestinationName)
- decorateHistoryDestinationsInternal(t.Context(), nil, history)
- require.Equal(t, "Offsite", history[0].S3DestinationName)
-}
diff --git a/backend/internal/system/children/upgrade/handler.go b/backend/internal/system/children/upgrade/handler.go
index 328cdbf94..12cf3f0ed 100644
--- a/backend/internal/system/children/upgrade/handler.go
+++ b/backend/internal/system/children/upgrade/handler.go
@@ -7,25 +7,23 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
// Handler serves the self-upgrade and update-all endpoints.
type Handler struct {
- service *Service
- environmentService *environment.EnvironmentService
- cfg *config.Config
- appCtx context.Context
+ service *Service
+ cfg *config.Config
+ appCtx context.Context
}
-func NewHandler(service *Service, environmentService *environment.EnvironmentService, cfg *config.Config, appCtx context.Context) *Handler {
- return &Handler{service: service, environmentService: environmentService, cfg: cfg, appCtx: appCtx}
+func NewHandler(service *Service, cfg *config.Config, appCtx context.Context) *Handler {
+ return &Handler{service: service, cfg: cfg, appCtx: appCtx}
}
type CheckUpgradeInput struct {
@@ -52,9 +50,8 @@ type TriggerUpgradeBody struct {
TargetVersion string `json:"targetVersion,omitempty" doc:"Release version to upgrade to; overrides this instance's own version check"`
}
-// TriggerUpgradeData reports the upgrade was accepted. UpToDate lets a client skip
-// waiting for a restart: the upgrader still pulls, but when the environment already
-// runs the newest image it finds nothing to swap in and no restart follows.
+// TriggerUpgradeData reports the upgrade was accepted. UpToDate lets a client skip waiting
+// for a restart: an environment already on the newest image is not recreated.
type TriggerUpgradeData struct {
Message string `json:"message" doc:"Response message"`
UpToDate bool `json:"upToDate" doc:"Environment already runs the newest image, so no restart is expected"`
@@ -68,14 +65,6 @@ type UpdateAllStatusInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
}
-// rejectIfAgentModeInternal blocks manager-only operations when running as an agent.
-func (h *Handler) rejectIfAgentModeInternal() error {
- if h.cfg != nil && h.cfg.AgentMode {
- return huma.Error400BadRequest("update-all is managed on the Arcane manager")
- }
- return nil
-}
-
// CheckUpgradeAvailable checks if a system upgrade is available.
func (h *Handler) CheckUpgradeAvailable(ctx context.Context, input *CheckUpgradeInput) (*CheckUpgradeOutput, error) {
canUpgrade, err := h.service.CanUpgrade(ctx)
@@ -108,16 +97,10 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput
slog.InfoContext(ctx, "System upgrade triggered", "user", user.Username, "userId", user.ID)
- // Resolved before triggering, while the version check still describes the running
- // container: the upgrade itself may replace it.
+ // Resolved before triggering, while the version check still describes the running container.
upToDate := h.service.AlreadyOnNewestImage(ctx)
- targetVersion := ""
- if input.Body != nil {
- targetVersion = input.Body.TargetVersion
- }
-
- err = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, targetVersion)
+ err = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body).TargetVersion)
if err != nil {
slog.ErrorContext(ctx, "System upgrade failed", "error", err, "user", user.Username)
@@ -145,11 +128,10 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput
}, nil
}
-// TriggerUpdateAll starts a fleet-wide update, upgrading the manager first and then
-// the remote agents (the latter resume after the manager restarts).
+// TriggerUpdateAll starts a fleet-wide update: every remote agent in turn, then the manager last.
func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllInput) (*handlerutil.Out[EnvironmentUpdateJob], error) {
- if err := h.rejectIfAgentModeInternal(); err != nil {
- return nil, err
+ if h.cfg != nil && h.cfg.AgentMode {
+ return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")
}
user, err := handlerutil.RequireUser(ctx)
@@ -159,11 +141,7 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI
slog.InfoContext(ctx, "Update-all environments triggered", "user", user.Username, "userId", user.ID)
- // Use a runtime context so the agents phase can outlive the request when the
- // manager is already up to date.
- runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
-
- job, err := h.service.StartUpdateAll(runtimeCtx, *user, h.environmentService)
+ job, err := h.service.StartUpdateAll(ctx, *user)
if err != nil {
if errors.Is(err, common.ErrUpdateAllInProgress) {
return nil, huma.Error409Conflict(err.Error())
@@ -181,8 +159,8 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI
// GetUpdateAllStatus returns the latest update-all job for live progress polling.
func (h *Handler) GetUpdateAllStatus(ctx context.Context, input *UpdateAllStatusInput) (*handlerutil.Out[EnvironmentUpdateJob], error) {
- if err := h.rejectIfAgentModeInternal(); err != nil {
- return nil, err
+ if h.cfg != nil && h.cfg.AgentMode {
+ return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")
}
job, err := h.service.GetLatestUpdateAllJob(ctx)
diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.go
index 0dfb3f05e..c713f8be3 100644
--- a/backend/internal/system/children/upgrade/model.go
+++ b/backend/internal/system/children/upgrade/model.go
@@ -12,9 +12,8 @@ import (
type EnvironmentUpdateJobStatus string
const (
- // EnvironmentUpdateJobStatusPendingRestart means the remote agents have been
- // updated and the manager has triggered its own self-upgrade as the final step;
- // it is waiting to restart on the new version, after which the job is finalized.
+ // EnvironmentUpdateJobStatusPendingRestart means the agents are done and the manager triggered its own
+ // self-upgrade; the job is finalized once it restarts on the new version.
EnvironmentUpdateJobStatusPendingRestart EnvironmentUpdateJobStatus = "pending_restart"
// EnvironmentUpdateJobStatusRunning means the agents phase is in progress.
EnvironmentUpdateJobStatusRunning EnvironmentUpdateJobStatus = "running"
@@ -23,9 +22,8 @@ const (
// EnvironmentUpdateJobStatusFailed means the job stopped before completing.
EnvironmentUpdateJobStatusFailed EnvironmentUpdateJobStatus = "failed"
- // EnvironmentUpdateResultStatusPending is the initial state recorded for an
- // environment that is seeded into the job but not yet being processed (and for
- // the manager entry while its self-upgrade is in flight).
+ // EnvironmentUpdateResultStatusPending is the initial state of an environment
+ // seeded into the job but not yet being processed.
EnvironmentUpdateResultStatusPending EnvironmentUpdateResultStatus = "pending"
// EnvironmentUpdateResultStatusUpdating means this environment is being processed
// right now; it drives the live per-environment progress indicator in the UI.
@@ -65,14 +63,15 @@ type EnvironmentUpdateResult struct {
EnvironmentName string `json:"environmentName"`
Status EnvironmentUpdateResultStatus `json:"status"`
FromVersion string `json:"fromVersion,omitempty"`
+ FromDigest string `json:"fromDigest,omitempty"`
ToVersion string `json:"toVersion,omitempty"`
Stage EnvironmentUpdateStage `json:"stage,omitempty"`
StageStartedAt *time.Time `json:"stageStartedAt,omitempty"`
Error string `json:"error,omitempty"`
}
-// clearStageInternal drops live-progress fields once the row leaves updating.
-func (r *EnvironmentUpdateResult) clearStageInternal() {
+// clearStage drops live-progress fields once the row leaves updating.
+func (r *EnvironmentUpdateResult) clearStage() {
r.Stage = ""
r.StageStartedAt = nil
}
@@ -91,9 +90,8 @@ func (r *EnvironmentUpdateResults) Scan(value any) error {
return database.JSONScan(r, value)
}
-// EnvironmentUpdateJob is a persisted fleet-wide update orchestration record. It
-// survives the manager's final self-upgrade restart so the manager result can be
-// finalized on the next boot. See [EnvironmentUpdateJobStatus].
+// EnvironmentUpdateJob is a persisted fleet-wide update record. It survives the manager's
+// self-upgrade restart so the next boot can finalize the manager result.
type EnvironmentUpdateJob struct {
database.BaseModel
diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.go
index b7f455536..2a3b3b034 100644
--- a/backend/internal/system/children/upgrade/service.go
+++ b/backend/internal/system/children/upgrade/service.go
@@ -19,15 +19,17 @@ import (
"github.com/containerd/errdefs"
"github.com/distribution/reference"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
versiontypes "github.com/getarcaneapp/arcane/types/v2/version"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
"github.com/opencontainers/go-digest"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
"go.getarcane.app/updater"
"go.getarcane.app/updater/labels"
@@ -40,27 +42,63 @@ import (
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
-var upgradeLogNameInternal = regexp.MustCompile(`^arcane-upgrade-\d+\.log$`)
+var (
+ upgradeLogName = regexp.MustCompile(`^arcane-upgrade-\d+\.log$`)
+ // exactReleaseTag pins an exact release (vX.Y.Z, prereleases included); channels like v2 or v2.9 are mutable.
+ exactReleaseTag = regexp.MustCompile(`^v?\d+\.\d+\.\d+(?:[-+].*)?$`)
+ activeUpdateAllStatuses = []EnvironmentUpdateJobStatus{EnvironmentUpdateJobStatusPendingRestart, EnvironmentUpdateJobStatusRunning}
+ errUpdateAllInProgress = common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))
+)
+
+const (
+ // The run never finished observably: nothing may be concluded from it.
+ upgraderExitUnobserved upgraderExit = 0
+ upgraderExitSucceeded upgraderExit = 1
+ upgraderExitFailed upgraderExit = 2
+ // The upgrader is gone and its exit code with it: success and failure are indistinguishable.
+ upgraderExitCodeLost upgraderExit = 3
+
+ updateAllStaleThreshold = time.Hour
+ updateAllAgentRequestTimeout = 15 * time.Second
+ updateAllConfirmPollInterval = 10 * time.Second
+ updateAllConfirmTimeout = 5 * time.Minute
+ updateAllErrorMaxLen = 500
+ updateAllWorkflowName = "update-all"
+ // Generous: the upgrader pulls the target image before recreating anything.
+ updateAllManagerWatchTimeout = 15 * time.Minute
+ // How long a recreate already underway gets to stop this container before no restart is assumed.
+ updateAllManagerNoRestartGrace = 15 * time.Second
+)
type Service struct {
- upgrading atomic.Bool
- updatingAll atomic.Bool
- db *database.DB
- dockerService *dockerInternal.DockerClientService
- versionService *version.VersionService
- eventService *event.EventService
- settingsService *settings.SettingsService
- projectService *project.ProjectService
+ upgrading atomic.Bool
+ updatingAll atomic.Bool
+ flow *flow.Engine
+ environments *environment.EnvironmentService
+ roles *role.RoleService
+ // updateAllWorkflow upgrades every agent in turn, then the manager last.
+ updateAllWorkflow *flow.Workflow
+ db *database.DB
+ dockerService *dockerInternal.DockerClientService
+ versionService *version.VersionService
+ eventService *event.EventService
+ settingsService *settings.SettingsService
+ projectService *project.ProjectService
// resolveRuntimeOptions determines how the upgrader container reaches the Docker daemon.
resolveRuntimeOptions func(
ctx context.Context,
@@ -72,6 +110,38 @@ type Service struct {
) ([]string, []mount.Mount, container.NetworkMode, error)
}
+// remoteAgent is one environment the update-all workflow upgrades.
+type remoteAgent struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+}
+
+// updateAllInput is the update-all payload: the job and the API key that requested it.
+type updateAllInput struct {
+ JobID string `json:"jobId"`
+ KeyID string `json:"keyId,omitempty"`
+}
+
+type preparedUpgrade struct {
+ dockerClient *client.Client
+ current container.InspectResponse
+ containerName string
+ binaryPath string
+ // targetImage is what the recreated container runs; pullImage is the immutable reference pulled (same unless frozen).
+ targetImage string
+ pullImage string
+ // saveCompose persists a tag change in the container's Compose project.
+ saveCompose func(context.Context)
+}
+
+type resumeAction struct {
+ markStale bool
+ managerSucceeded bool
+}
+
+// upgraderExit is how much the watcher learned about an upgrader run.
+type upgraderExit int
+
func NewService(
db *database.DB,
dockerService *dockerInternal.DockerClientService,
@@ -99,56 +169,59 @@ func NewService(
}
}
-// CanUpgrade checks if self-upgrade is possible
-func (s *Service) CanUpgrade(ctx context.Context) (bool, error) {
- // Check if running in Docker
- containerId, err := s.getCurrentContainerIDInternal(ctx)
- if err != nil {
- return false, err
- }
+// RegisterWorkflows defines the update-all workflow while the host is still unstarted. Its payload names the job;
+// the job row holds every result, so a redelivered task resumes from what the row records.
+func (s *Service) RegisterWorkflows(engine *flow.Engine, environments *environment.EnvironmentService, roles *role.RoleService) error {
+ var err error
+ s.flow, s.environments, s.roles = engine, environments, roles
+ s.updateAllWorkflow, err = engine.Define(flow.Definition{
+ Name: updateAllWorkflowName,
+ Version: 1,
+ Fingerprint: "65b33dbbe55a5018517b2e8506d9b487f322803ed5a9dfec0edbf5103c96e586",
+ Concurrency: 1,
+ Timeout: 24 * time.Hour,
+ Steps: []workflow.StepSpec{
+ workflow.Step("discover", engine.Handler(s.discoverAgents), workflow.WithMaxAttempts(1)),
+ workflow.ForEach("agents", engine.Handler(s.upgradeAgentTask), workflow.WithItemsFrom("discover"), workflow.WithMaxParallel(1),
+ workflow.WithMaxAttempts(1), workflow.WithFailurePolicy(workflow.TolerateFailures)),
+ workflow.Step("manager", engine.Handler(s.upgradeManagerTask), workflow.WithMaxAttempts(1)),
+ },
+ })
+ return err
+}
- // Verify we can access Docker
- _, err = s.dockerService.GetClient(ctx)
+// CanUpgrade checks if self-upgrade is possible.
+func (s *Service) CanUpgrade(ctx context.Context) (bool, error) {
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return false, errors.New("docker socket is not accessible")
}
-
- // Verify we can find our container
- _, err = s.findArcaneContainerInternal(ctx, containerId)
- if err != nil {
- return false, err
+ if _, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr != nil {
+ return false, errors.New("arcane is not running in a Docker container")
}
-
return true, nil
}
-// AlreadyOnNewestImage reports whether this environment's version check is confident
-// it already runs the newest image. A triggered upgrade still pulls, but will then
-// find nothing to swap in and skip the recreate — so callers can use this to stop
-// waiting for a restart that is not coming.
+// AlreadyOnNewestImage reports whether the version check is confident this environment runs the newest image,
+// so a triggered upgrade will find nothing to swap in and no restart will follow.
func (s *Service) AlreadyOnNewestImage(ctx context.Context) bool {
return s.versionService.GetAppVersionInfo(ctx).AlreadyOnNewest()
}
-// TriggerUpgradeViaCLI spawns the upgrade CLI command in a separate container and
-// returns that upgrader container's ID. This avoids self-termination issues by running
-// the upgrade from outside. A zero-value target resolves the image to upgrade to from
-// the version check (see resolveSelfUpgradeTargetImageInternal); the updater engine
-// passes an explicit target with the resolved new image, which is used as-is.
-// Update-all uses the returned ID to tell an upgrade that recreated this
-// container from one that found nothing to do — see watchManagerUpgraderInternal.
+// TriggerUpgradeViaCLI runs the upgrade from a separate upgrader container and returns its ID.
+// A zero-value target resolves the image from the version check; an explicit target is used as-is.
func (s *Service) TriggerUpgradeViaCLI(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {
- prepared, err := s.prepareUpgradeInternal(ctx, user, target, "")
+ prepared, err := s.prepareUpgrade(ctx, user, target, "")
if err != nil {
return "", err
}
- return s.runPreparedUpgradeInternal(ctx, prepared)
+ return s.runPreparedUpgrade(ctx, prepared)
}
// TriggerUpgradeAsync validates synchronously, then pulls and spawns the upgrader in the background (#3628).
// The run follows the app lifecycle with a deadline so a stalled daemon cannot hold the upgrading guard forever.
func (s *Service) TriggerUpgradeAsync(ctx context.Context, user usertypes.Actor, targetVersion string) error {
- prepared, err := s.prepareUpgradeInternal(ctx, user, updater.SelfUpdateTarget{}, targetVersion)
+ prepared, err := s.prepareUpgrade(ctx, user, updater.SelfUpdateTarget{}, targetVersion)
if err != nil {
return err
}
@@ -157,27 +230,15 @@ func (s *Service) TriggerUpgradeAsync(ctx context.Context, user usertypes.Actor,
runCtx, cancel := context.WithTimeout(utils.ActivityRuntimeContext(ctx, nil), runTimeout)
go func() {
defer cancel()
- if _, runPreparedUpgradeErr := s.runPreparedUpgradeInternal(runCtx, prepared); runPreparedUpgradeErr != nil {
+ if _, runPreparedUpgradeErr := s.runPreparedUpgrade(runCtx, prepared); runPreparedUpgradeErr != nil {
slog.ErrorContext(ctx, "Background self-upgrade failed", "error", runPreparedUpgradeErr, "targetImage", prepared.targetImage)
}
}()
return nil
}
-type preparedUpgradeInternal struct {
- current container.InspectResponse
- containerName string
- binaryPath string
- // targetImage is the reference the recreated container runs as; pullImage
- // is the immutable reference actually pulled (the same unless frozen).
- targetImage string
- pullImage string
- // saveCompose persists a tag change in the container's Compose project.
- saveCompose func(context.Context)
-}
-
-// prepareUpgradeInternal takes the upgrading guard, released by runPreparedUpgradeInternal (or here on error).
-func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget, targetVersion string) (prepared *preparedUpgradeInternal, err error) {
+// prepareUpgrade takes the upgrading guard, released by runPreparedUpgrade (or here on error).
+func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget, targetVersion string) (prepared *preparedUpgrade, err error) {
if !s.upgrading.CompareAndSwap(false, true) {
return nil, common.Classify(common.ErrUpgradeInProgress, errors.New("an upgrade is already in progress"))
}
@@ -187,42 +248,58 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Act
}
}()
- containerId := strings.TrimSpace(target.ContainerID)
- if containerId == "" {
- // Fall back to the container this process runs in
- containerId, err = s.getCurrentContainerIDInternal(ctx)
- if err != nil {
- return nil, fmt.Errorf("get current container: %w", err)
- }
- }
-
- currentContainer, err := s.findArcaneContainerInternal(ctx, containerId)
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return nil, fmt.Errorf("inspect container: %w", err)
+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+ var current container.InspectResponse
+ if containerID := strings.TrimSpace(target.ContainerID); containerID != "" {
+ inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ if inspectErr != nil {
+ // Fall back to a prefix match across every container.
+ containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ if listErr != nil {
+ return nil, fmt.Errorf("inspect container: %w", listErr)
+ }
+ idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })
+ if idx < 0 {
+ return nil, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))
+ }
+ if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {
+ return nil, fmt.Errorf("inspect container: %w", inspectErr)
+ }
+ }
+ current = inspect.Container
+ } else {
+ self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ if inspectErr != nil {
+ return nil, errors.New("get current container: arcane is not running in a Docker container")
+ }
+ current = *self
}
- containerName := strings.TrimPrefix(currentContainer.Name, "/")
-
- // Determine binary path based on container type (agent vs main)
- binaryPath := "/app/arcane"
- if currentContainer.Config != nil {
- binaryPath = kit.Ternary(labels.IsArcaneAgentContainer(currentContainer.Config.Labels), "/app/arcane-agent", "/app/arcane")
- }
+ containerName := strings.TrimPrefix(current.Name, "/")
+ config := kit.FromPtr(current.Config)
+ binaryPath := kit.Ternary(labels.IsArcaneAgentContainer(config.Labels), "/app/arcane-agent", "/app/arcane")
- targetImage, err := s.resolveUpgradeTargetImageInternal(ctx, currentContainer, target.NewImageRef, targetVersion)
- if err != nil {
- return nil, err
+ targetImage := strings.TrimSpace(target.NewImageRef)
+ if targetImage == "" {
+ // A blank target resolves against the version check; a manager-supplied version outranks this instance's own.
+ info := kit.FromPtr(s.versionService.GetAppVersionInfo(ctx))
+ info.NewestVersion = cmp.Or(strings.TrimSpace(targetVersion), info.NewestVersion)
+ if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &info); err != nil {
+ return nil, fmt.Errorf("resolve upgrade target image: %w", err)
+ }
}
- targetImage, saveCompose, err := s.configuredTargetImageInternal(ctx, currentContainer, targetImage)
+ targetImage, saveCompose, err := s.configuredTargetImage(ctx, current, targetImage)
if err != nil {
return nil, err
}
pullImage := cmp.Or(strings.TrimSpace(target.PullImageRef), targetImage)
- // Log upgrade event
metadata := database.JSON{
"action": "system_upgrade_cli",
- "containerId": containerId,
+ "containerId": current.ID,
"containerName": containerName,
"method": "cli",
"targetImage": targetImage,
@@ -232,8 +309,9 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Act
slog.WarnContext(ctx, "Failed to log upgrade event", "error", logUserEventErr)
}
- return &preparedUpgradeInternal{
- current: currentContainer,
+ return &preparedUpgrade{
+ dockerClient: dockerClient,
+ current: current,
containerName: containerName,
binaryPath: binaryPath,
targetImage: targetImage,
@@ -242,34 +320,21 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Act
}, nil
}
-func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *preparedUpgradeInternal) (string, error) {
+func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgrade) (string, error) {
defer s.upgrading.Store(false)
- // Run the upgrader from the image we are upgrading to, so the upgrade CLI
- // is the new version.
- upgraderImage := prepared.pullImage
- slog.DebugContext(ctx, "Using upgrader image", "image", upgraderImage)
-
- slog.InfoContext(ctx, "Spawning upgrade CLI command", "containerName", prepared.containerName, "upgraderImage", upgraderImage)
-
- // Spawn the upgrade command in a detached container
- // This will run independently of the current container
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- // Pull the upgrader image first to ensure it exists
- slog.InfoContext(ctx, "Pulling upgrader image", "image", upgraderImage)
+ // The upgrader runs from the target image, so the upgrade CLI is the new version.
+ dockerClient := prepared.dockerClient
+ slog.InfoContext(ctx, "Pulling upgrader image", "containerName", prepared.containerName, "image", prepared.pullImage)
localSettings := s.settingsService.GetSettingsConfig()
pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
defer pullCancel()
- pullReader, err := dockerClient.ImagePull(pullCtx, upgraderImage, client.ImagePullOptions{})
+ pullReader, err := dockerClient.ImagePull(pullCtx, prepared.pullImage, client.ImagePullOptions{})
if err != nil {
if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {
- return "", fmt.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", upgraderImage)
+ return "", fmt.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", prepared.pullImage)
}
return "", fmt.Errorf("pull upgrader image: %w", err)
}
@@ -281,12 +346,12 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep
if closeErr := pullReader.Close(); closeErr != nil {
slog.WarnContext(ctx, "Failed to close upgrader image pull reader", "error", closeErr)
}
- slog.InfoContext(ctx, "Upgrader image pulled successfully", "image", upgraderImage)
+ slog.InfoContext(ctx, "Upgrader image pulled successfully", "image", prepared.pullImage)
if prepared.saveCompose != nil {
prepared.saveCompose(ctx)
}
- // Try to get the /app/data mount from current container so upgrade logs persist.
+ // Mount /app/data from the current container so upgrade logs persist.
appDataMount := docker.MountForDestination(prepared.current.Mounts, libarcane.UpgradeLogDirectory, libarcane.UpgradeLogDirectory)
if appDataMount == nil {
slog.WarnContext(ctx, "Could not detect /app/data mount; upgrader logs may not persist")
@@ -294,7 +359,6 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep
slog.DebugContext(ctx, "Mounting /app/data into upgrader container", "type", appDataMount.Type, "source", appDataMount.Source)
}
- // Create the upgrader container config
containerEnv, runtimeMounts, networkMode, err := s.resolveRuntimeOptions(
ctx,
s.dockerService.DockerHost(),
@@ -323,11 +387,9 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep
}
config := &container.Config{
- Image: upgraderImage,
+ Image: prepared.pullImage,
Cmd: upgradeCmd,
- // The upgrader needs root for the Docker socket; unlike the server it
- // is short-lived and never goes through the runtime-identity drop, so
- // don't rely on the image's default user.
+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.
User: "0:0",
Env: containerEnv,
Labels: map[string]string{
@@ -341,32 +403,33 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep
mounts = append(mounts, *appDataMount)
}
- keepUpgraderContainer := strings.EqualFold(strings.TrimSpace(os.Getenv("ARCANE_UPGRADE_KEEP_CONTAINER")), "true")
+ keepUpgraderContainer, _ := kit.ParseBool(os.Getenv("ARCANE_UPGRADE_KEEP_CONTAINER"))
if keepUpgraderContainer {
slog.InfoContext(ctx, "Keeping upgrader container after exit (ARCANE_UPGRADE_KEEP_CONTAINER=true)")
}
hostConfig := &container.HostConfig{
- AutoRemove: !keepUpgraderContainer, // default: clean up after completion
+ AutoRemove: !keepUpgraderContainer,
Mounts: mounts,
NetworkMode: networkMode,
}
- // Inherit the security context that lets the running Arcane container reach
- // the Docker socket (e.g. SELinux label=disable, privileged); the upgrader
- // needs the same access on hardened hosts.
+ // Inherit the security context (SELinux label, privileged) that lets Arcane reach the socket on hardened hosts.
if prepared.current.HostConfig != nil {
hostConfig.SecurityOpt = slices.Clone(prepared.current.HostConfig.SecurityOpt)
hostConfig.Privileged = prepared.current.HostConfig.Privileged
}
- // On SELinux-enforcing hosts the socket carries container_var_run_t, which
- // container processes cannot connect to regardless of UID; without an
- // explicit label opt the upgrader would exit with EACCES and auto-remove.
- if !hostConfig.Privileged && !hasSELinuxLabelOptInternal(hostConfig.SecurityOpt) && daemonHasSELinuxEnabledInternal(ctx, dockerClient) {
- hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")
+ // On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.
+ hasLabelOpt := slices.ContainsFunc(hostConfig.SecurityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") })
+ if !hostConfig.Privileged && !hasLabelOpt {
+ infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})
+ if infoErr != nil {
+ slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)
+ } else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {
+ hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")
+ }
}
upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())
-
resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: config,
HostConfig: hostConfig,
@@ -376,42 +439,18 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep
return "", fmt.Errorf("create upgrader container: %w", err)
}
- // Start the upgrader container - it will run the upgrade and auto-remove
if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
_, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
return "", fmt.Errorf("start upgrader container: %w", containerStartErr)
}
slog.InfoContext(ctx, "Upgrade container started", "upgraderId", resp.ID[:12], "upgraderName", upgraderName)
-
return resp.ID, nil
}
-// hasSELinuxLabelOptInternal reports whether the security options already set
-// an SELinux label policy (e.g. "label=disable", "label:disable", "label=type:...").
-func hasSELinuxLabelOptInternal(securityOpts []string) bool {
- for _, opt := range securityOpts {
- if strings.HasPrefix(strings.TrimSpace(opt), "label") {
- return true
- }
- }
- return false
-}
-
-func daemonHasSELinuxEnabledInternal(ctx context.Context, dockerClient *client.Client) bool {
- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})
- if err != nil {
- slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", err)
- return false
- }
- return slices.Contains(infoResult.Info.SecurityOptions, "name=selinux")
-}
-
-// configuredTargetImageInternal spells targetImage like the container's Compose
-// service, or like its runtime image when Arcane does not manage that project.
-// A tag change also returns the Compose edit to save before the upgrader starts;
-// sources Arcane cannot edit keep upgrading without it, as they did before.
-func (s *Service) configuredTargetImageInternal(ctx context.Context, current container.InspectResponse, targetImage string) (string, func(context.Context), error) {
+// configuredTargetImage spells targetImage like the container's Compose service (or its runtime image when unmanaged).
+// A tag change also returns the Compose edit to save before the upgrader starts.
+func (s *Service) configuredTargetImage(ctx context.Context, current container.InspectResponse, targetImage string) (string, func(context.Context), error) {
// Digest targets cannot be written as Compose tags.
if refs.NormalizeImageUpdateRef(targetImage) == "" || current.Config == nil {
return targetImage, nil, nil
@@ -444,43 +483,9 @@ func (s *Service) configuredTargetImageInternal(ctx context.Context, current con
}, nil
}
-// resolveUpgradeTargetImageInternal picks the image the upgrade should move to.
-// Explicit targets from the updater engine are authoritative. A blank target
-// (manual trigger, update-all) resolves against the version check so a
-// version-pinned install actually moves to the newest release (#3687).
-func (s *Service) resolveUpgradeTargetImageInternal(ctx context.Context, currentContainer container.InspectResponse, explicitImageRef, targetVersion string) (string, error) {
- if targetImage := strings.TrimSpace(explicitImageRef); targetImage != "" {
- return targetImage, nil
- }
-
- currentImageRef := ""
- if currentContainer.Config != nil {
- currentImageRef = strings.TrimSpace(currentContainer.Config.Image)
- }
- info := s.versionService.GetAppVersionInfo(ctx)
- // A manager-supplied target version outranks this instance's own version check.
- if targetVersion = strings.TrimSpace(targetVersion); targetVersion != "" {
- merged := versiontypes.Info{}
- if info != nil {
- merged = *info
- }
- merged.NewestVersion = targetVersion
- info = &merged
- }
- resolved, err := resolveSelfUpgradeTargetImageInternal(currentImageRef, info)
- if err != nil {
- return "", fmt.Errorf("resolve upgrade target image: %w", err)
- }
- return resolved, nil
-}
-
-// resolveSelfUpgradeTargetImageInternal resolves the image for a blank-target
-// self-upgrade from the running container's reference and the version check.
-// Exact release tags (vX.Y.Z) move to the newest release; mutable channels and
-// untagged references keep theirs; digest pins move only to a resolved newest
-// digest. Unresolved or older exact-version targets fail instead of
-// reinstalling or downgrading (#3687).
-func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *versiontypes.Info) (string, error) {
+// resolveSelfUpgradeTargetImage moves exact release tags to the newest release and digest pins to the newest digest;
+// mutable channels keep their reference. Unresolved or older exact-version targets fail (#3687).
+func resolveSelfUpgradeTargetImage(currentImageRef string, info *versiontypes.Info) (string, error) {
currentImageRef = strings.TrimSpace(currentImageRef)
if currentImageRef == "" {
return "", errors.New("running container has no image reference to upgrade from")
@@ -495,10 +500,9 @@ func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *version
return "", fmt.Errorf("current image reference %q is not a named image", currentImageRef)
}
- newestDigest, newestVersion := newestTargetIdentifiersInternal(info)
+ newest := kit.FromPtr(info)
+ newestDigest, newestVersion := strings.TrimSpace(newest.NewestDigest), strings.TrimSpace(newest.NewestVersion)
- // Digest-pinned installs only move when the version check resolved a new
- // digest; otherwise there is nothing to point them at.
if _, isDigested := parsed.(reference.Digested); isDigested {
if newestDigest == "" {
return "", errors.New("image is digest-pinned but no newest digest could be resolved; refusing an upgrade that could not move it")
@@ -515,232 +519,106 @@ func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *version
}
tagged, isTagged := parsed.(reference.Tagged)
- if !isTagged || !isExactReleaseTagInternal(tagged.Tag()) {
- // Mutable channel or untagged: the pull re-resolves whatever the name
- // points at, so keeping the reference advances the digest in place.
+ if !isTagged || !exactReleaseTag.MatchString(tagged.Tag()) {
+ // The pull re-resolves a mutable channel, so keeping the reference advances the digest in place.
return currentImageRef, nil
}
if newestVersion == "" {
return "", fmt.Errorf("running exact release %q but the newest release could not be resolved", tagged.Tag())
}
- newest := kit.EnsurePrefix(newestVersion, "v")
- if !semver.IsValid(newest) {
+ newestTag := kit.EnsurePrefix(newestVersion, "v")
+ if !semver.IsValid(newestTag) {
return "", fmt.Errorf("resolved newest version %q is not a valid semver release", newestVersion)
}
- if semver.Compare(newest, kit.EnsurePrefix(tagged.Tag(), "v")) < 0 {
+ if semver.Compare(newestTag, kit.EnsurePrefix(tagged.Tag(), "v")) < 0 {
return "", fmt.Errorf("newest release %q is older than the running %q; refusing to downgrade", newestVersion, tagged.Tag())
}
- withTag, err := reference.WithTag(named, newest)
+ withTag, err := reference.WithTag(named, newestTag)
if err != nil {
return "", fmt.Errorf("build target reference for %q: %w", named.Name(), err)
}
return withTag.String(), nil
}
-// isExactReleaseTagInternal reports whether tag pins an exact release version
-// (vX.Y.Z, prereleases included). Short channels like v2 or v2.9 are mutable.
-// The x/mod/semver parser is deliberately lenient (it accepts v2 and v2.9), so
-// the three numeric components are checked explicitly.
-func isExactReleaseTagInternal(tag string) bool {
- core := strings.TrimPrefix(strings.TrimSpace(tag), "v")
- if i := strings.IndexAny(core, "-+"); i >= 0 {
- core = core[:i]
- }
- parts := strings.Split(core, ".")
- if len(parts) != 3 {
- return false
- }
- for _, part := range parts {
- if part == "" {
- return false
- }
- for _, c := range part {
- if c < '0' || c > '9' {
- return false
- }
- }
- }
- return true
-}
-
-// newestTargetIdentifiersInternal reads the version-check result nil-safely: the
-// service may be unavailable and the check itself may resolve neither identifier.
-func newestTargetIdentifiersInternal(info *versiontypes.Info) (newestDigest, newestVersion string) {
- if info == nil {
- return "", ""
- }
- return strings.TrimSpace(info.NewestDigest), strings.TrimSpace(info.NewestVersion)
-}
-
-// getCurrentContainerID detects if we're running in Docker and returns container ID
-func (s *Service) getCurrentContainerIDInternal(ctx context.Context) (string, error) {
- // cgroup detection fails on cgroupv2 with a private namespace, and with
- // network_mode: service:<sidecar> the hostname identifies the sidecar —
- // InspectCurrentArcaneContainer adds the Arcane-label fallback (#3544).
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return "", err
- }
- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
- if err != nil {
- return "", errors.New("arcane is not running in a Docker container")
- }
- return inspect.ID, nil
-}
-
-// findArcaneContainer finds the container using the ID
-func (s *Service) findArcaneContainerInternal(ctx context.Context, containerId string) (container.InspectResponse, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return container.InspectResponse{}, err
- }
-
- // Try to inspect the container directly
- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerId, client.ContainerInspectOptions{})
- if err == nil {
- return inspectResult.Container, nil
- }
-
- // Fallback: search for containers with arcane image
- filter := make(client.Filters)
- filter = filter.Add("ancestor", "ghcr.io/getarcaneapp/arcane")
-
- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{
- All: true,
- Filters: filter,
- })
- if err != nil {
- return container.InspectResponse{}, err
- }
-
- for _, c := range containers.Items {
- if strings.HasPrefix(c.ID, containerId) {
- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})
- if inspectErr != nil {
- return container.InspectResponse{}, inspectErr
- }
- return inspect.Container, nil
- }
- }
-
- // Try without filter - search all containers
- allContainers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
- if err != nil {
- return container.InspectResponse{}, err
- }
-
- for _, c := range allContainers.Items {
- if strings.HasPrefix(c.ID, containerId) || c.ID == containerId {
- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})
- if inspectErr != nil {
- return container.InspectResponse{}, inspectErr
- }
- return inspect.Container, nil
- }
- }
-
- return container.InspectResponse{}, common.Classify(common.ErrNotFound, errors.New("could not find Arcane container"))
-}
-
-const (
- // The run never finished observably: nothing may be concluded from it.
- upgraderExitUnobservedInternal upgraderExitInternal = 0
- // The upgrader exited 0.
- upgraderExitSucceededInternal upgraderExitInternal = 1
- // The upgrader exited non-zero.
- upgraderExitFailedInternal upgraderExitInternal = 2
- // The upgrader is gone, but its exit code was lost with it — success and failure
- // are indistinguishable.
- upgraderExitCodeLostInternal upgraderExitInternal = 3
-
- updateAllStaleThresholdInternal = time.Hour
- updateAllAgentRequestTimeoutInternal = 15 * time.Second
- updateAllConfirmPollIntervalInternal = 10 * time.Second
- updateAllConfirmTimeoutInternal = 5 * time.Minute
- updateAllErrorMaxLenInternal = 500
-
- // How long to watch the manager's upgrader container for an exit. Generous: it
- // pulls the target image before recreating anything.
- updateAllManagerWatchTimeoutInternal = 15 * time.Minute
- // How long to let a recreate that is already underway stop this container before
- // concluding the manager was up to date and no restart is coming.
- updateAllManagerNoRestartGraceInternal = 15 * time.Second
-)
-
-// StartUpdateAll begins a fleet-wide update. The agents phase runs first in the
-// background (while the manager is up); the agents goroutine triggers the manager
-// self-upgrade as its final step (job left pending_restart, finalized at next
-// boot). Every environment pulls the latest image, whether or not it reports an
-// update available.
-func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor, env *environment.EnvironmentService) (*EnvironmentUpdateJob, error) {
- // Guard the check-then-create against concurrent callers (e.g. a double-click).
- // A dedicated flag rather than s.upgrading, because the manager branch below
- // calls TriggerUpgradeViaCLI which acquires s.upgrading itself. The persisted
- // job row is the durable guard once committed; this only closes the in-process
- // window before that row exists.
+// StartUpdateAll records an update-all job and submits the workflow that upgrades every agent, then the manager.
+// Every environment pulls the latest image, whether or not it reports an update.
+func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*EnvironmentUpdateJob, error) {
+ // Closes the in-process check-then-create window (e.g. a double-click); the job row is the durable guard.
if !s.updatingAll.CompareAndSwap(false, true) {
- return nil, common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))
+ return nil, errUpdateAllInProgress
}
defer s.updatingAll.Store(false)
- active, err := s.activeUpdateAllJobInternal(ctx)
+ active, err := s.findUpdateAllJob(ctx, "status IN ?", activeUpdateAllStatuses)
if err != nil {
return nil, fmt.Errorf("check for active update-all job: %w", err)
}
if active != nil {
- return nil, common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))
+ return nil, errUpdateAllInProgress
}
info := s.versionService.GetAppVersionInfo(ctx)
-
- managerResult := EnvironmentUpdateResult{
- EnvironmentID: environment.LocalEnvironmentID,
- EnvironmentName: env.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),
- FromVersion: info.CurrentVersion,
- }
-
- // Seed a pending row for every remote environment up front so the dialog can
- // show the whole fleet immediately instead of popping rows in as each finishes.
- // Best effort: the agents phase re-lists authoritatively and fills any gaps.
- remoteResults := s.seedRemoteResultsInternal(ctx, env)
-
job := &EnvironmentUpdateJob{
+ Status: EnvironmentUpdateJobStatusRunning,
UserID: user.ID,
Username: user.Username,
ManagerVersionAtStart: info.CurrentVersion,
ManagerDigestAtStart: info.CurrentDigest,
- ManagerTargetVersion: updateAllTargetVersionInternal(info),
+ ManagerTargetVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),
+ }
+ // The manager row stays pending until the final manager step.
+ job.Results = EnvironmentUpdateResults{{
+ EnvironmentID: environment.LocalEnvironmentID,
+ EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),
+ Status: EnvironmentUpdateResultStatusPending,
+ FromVersion: info.CurrentVersion,
+ ToVersion: job.ManagerTargetVersion,
+ }}
+ // Seed every remote row so the dialog shows the whole fleet at once; the agents step fills any gaps.
+ if remotes, listErr := s.environments.ListRemoteEnvironments(ctx); listErr != nil {
+ slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", listErr)
+ } else {
+ for _, remote := range remotes {
+ upsertPendingResult(job, remote.ID, remote.Name)
+ }
}
-
- // Manager upgrades LAST. Seed its row pending; the agents-phase goroutine
- // flips it to updating right before triggering the self-upgrade.
- managerResult.Status = EnvironmentUpdateResultStatusPending
- managerResult.ToVersion = job.ManagerTargetVersion
-
- // Running from the start: the agents phase happens first, while the backend is
- // up and can report progress. The manager self-upgrade fires at the very end of
- // the agents goroutine.
- job.Status = EnvironmentUpdateJobStatusRunning
- job.Results = append(EnvironmentUpdateResults{managerResult}, remoteResults...)
if createUpgradeJobErr := s.db.WithContext(ctx).Create(job).Error; createUpgradeJobErr != nil {
return nil, fmt.Errorf("create update-all job: %w", createUpgradeJobErr)
}
+ input := updateAllInput{JobID: job.ID}
+ input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, input, activitylib.StartRequest{})
+ if submitErr != nil {
+ s.markUpdateAllFailed(context.WithoutCancel(ctx), job, fmt.Sprintf("failed to start update-all: %v", submitErr))
+ return nil, fmt.Errorf("start update-all: %w", submitErr)
+ }
+ go s.watchUpdateAll(context.WithoutCancel(ctx), job.ID, instanceID)
slog.InfoContext(ctx, "Update-all started; upgrading agents first", "jobId", job.ID, "user", user.Username)
- go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env, user)
-
return job, nil
}
-// ResumeUpdateAllOnStartup is called once at manager startup. When the manager
-// self-upgraded as the final step of an update-all (job left pending_restart), the
-// agents phase already ran before the restart — so this finalizes the manager's own
-// result and closes the job. It is a no-op when there is nothing pending.
+// watchUpdateAll fails a job whose workflow ends without reaching the manager step, which would leave it running
+// and refuse every later run.
+func (s *Service) watchUpdateAll(ctx context.Context, jobID, instanceID string) {
+ outcome, err := s.flow.Wait(ctx, s.updateAllWorkflow, instanceID)
+ if err != nil || outcome.Status == scheduler.Succeeded {
+ return
+ }
+ current, err := s.loadUpdateAllJob(ctx, jobID)
+ if err != nil || current.Status != EnvironmentUpdateJobStatusRunning {
+ return
+ }
+ s.markUpdateAllFailed(ctx, current, cmp.Or(outcome.Message, "update-all did not finish"))
+}
+
+// ResumeUpdateAllOnStartup finalizes a job the manager left pending_restart by self-upgrading,
+// and fails a running job whose workflow is gone. It is a no-op when nothing is pending.
func (s *Service) ResumeUpdateAllOnStartup(ctx context.Context) {
- job, err := s.activeUpdateAllJobInternal(ctx)
+ job, err := s.findUpdateAllJob(ctx, "status IN ?", activeUpdateAllStatuses)
if err != nil {
slog.WarnContext(ctx, "Failed to load pending update-all job on startup", "error", err)
return
@@ -749,492 +627,382 @@ func (s *Service) ResumeUpdateAllOnStartup(ctx context.Context) {
return
}
- // A job left running means the manager died mid-agents-phase, before it reached
- // the manager step; we can't safely resume partial progress, so fail it.
+ // A running job resumes with its workflow; one without a live workflow cannot resume, so fail it.
if job.Status == EnvironmentUpdateJobStatusRunning {
- s.markUpdateAllFailedInternal(ctx, job, "interrupted by manager restart")
+ standalone, standaloneErr := s.flow.Standalone(ctx)
+ if index := slices.IndexFunc(standalone, func(run scheduler.Run) bool { return run.JobID == updateAllWorkflowName }); standaloneErr == nil && index >= 0 {
+ go s.watchUpdateAll(context.WithoutCancel(ctx), job.ID, standalone[index].ID)
+ return
+ }
+ s.markUpdateAllFailed(ctx, job, "interrupted by manager restart")
return
}
info := s.versionService.GetAppVersionInfo(ctx)
- action := resolveResumeActionInternal(job, info.CurrentVersion, info.CurrentDigest, time.Now())
-
+ action := resolveResumeAction(job, info.CurrentVersion, info.CurrentDigest, time.Now())
if action.markStale {
- s.markUpdateAllFailedInternal(ctx, job, "update-all job is stale; manager did not restart in time")
+ s.markUpdateAllFailed(ctx, job, "update-all job is stale; manager did not restart in time")
return
}
- // The agents phase already ran before the restart. Finalize the manager's own
- // result and close the job — do not re-run the agents phase.
+ // The agents ran before the restart; only the manager row is left to settle.
managerStatus := kit.Ternary(action.managerSucceeded, EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusFailed)
- s.recordManagerResultInternal(job, managerStatus, info.CurrentVersion)
- s.finalizeUpdateAllJobInternal(ctx, job)
-
+ s.finalizeUpdateAllJob(ctx, job, managerStatus, info.CurrentVersion)
slog.InfoContext(ctx, "Finalized update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)
}
-type resumeActionInternal struct {
- markStale bool
- managerSucceeded bool
+// resolveResumeAction marks a pending_restart job stale after the threshold, otherwise reports whether the manager upgrade landed.
+func resolveResumeAction(job *EnvironmentUpdateJob, currentVersion, currentDigest string, now time.Time) resumeAction {
+ if now.Sub(job.CreatedAt) > updateAllStaleThreshold {
+ return resumeAction{markStale: true}
+ }
+ landed := upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)
+ return resumeAction{managerSucceeded: landed}
}
-// resolveResumeActionInternal is the pure decision for a resumed pending_restart
-// job: stale if it has waited too long, otherwise the manager upgrade is considered
-// successful when either the version or the digest changed from the at-start values
-// (digest covers non-semver, digest-pinned installs), or when the manager is already
-// on the recorded target — a force-update of an up-to-date manager recreates the
-// container without changing either.
-func resolveResumeActionInternal(job *EnvironmentUpdateJob, currentVersion, currentDigest string, now time.Time) resumeActionInternal {
- if now.Sub(job.CreatedAt) > updateAllStaleThresholdInternal {
- return resumeActionInternal{markStale: true}
- }
-
- versionChanged := job.ManagerVersionAtStart != "" && currentVersion != job.ManagerVersionAtStart
- digestChanged := job.ManagerDigestAtStart != "" && currentDigest != job.ManagerDigestAtStart
- // ManagerTargetVersion holds the newest version tag when known, otherwise the
- // newest digest — compare against both current identifiers.
- onTarget := job.ManagerTargetVersion != "" &&
- (strings.TrimPrefix(currentVersion, "v") == strings.TrimPrefix(job.ManagerTargetVersion, "v") ||
- currentDigest == job.ManagerTargetVersion)
-
- return resumeActionInternal{managerSucceeded: versionChanged || digestChanged || onTarget}
+// upgradeLanded reports whether the version or digest moved off its baseline, or already matches target (a version or
+// digest), so a force-update that recreates the same image still counts.
+func upgradeLanded(fromVersion, fromDigest, target, currentVersion, currentDigest string) bool {
+ versionChanged := fromVersion != "" && currentVersion != fromVersion
+ // An agent whose Docker lookup failed reports no digest, which proves nothing.
+ digestChanged := fromDigest != "" && currentDigest != "" && currentDigest != fromDigest
+ onTarget := target != "" && (strings.TrimPrefix(currentVersion, "v") == strings.TrimPrefix(target, "v") || currentDigest == target)
+ return versionChanged || digestChanged || onTarget
}
-// runAgentsPhaseInternal upgrades every online remote environment sequentially,
-// persisting progress after each one so the status endpoint can report live
-// progress, then triggers the manager's own self-upgrade as the final step
-// (leaving the job pending_restart for the next boot to finalize).
-func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env *environment.EnvironmentService, user usertypes.Actor) {
- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)
- if err != nil || job == nil {
- slog.WarnContext(ctx, "update-all: failed to reload job for agents phase", "jobId", jobID, "error", err)
- return
- }
-
- envs, err := env.ListRemoteEnvironments(ctx)
+// discoverAgents lists the remote environments the agents step upgrades in turn.
+func (s *Service) discoverAgents(ctx context.Context, t flow.Task) (any, error) {
+ envs, err := s.environments.ListRemoteEnvironments(ctx)
if err != nil {
- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("failed to list remote environments: %v", err))
- return
- }
-
- for _, remote := range envs {
- // Find the row seeded at job start (or append one if seeding missed this
- // environment) and mark it updating so the dialog shows a live indicator on
- // the row currently being processed.
- idx := upsertPendingResultInternal(job, remote.ID, remote.Name)
- job.Results[idx].Status = EnvironmentUpdateResultStatusUpdating
- if persistUpdateAllJobErr := s.persistUpdateAllJobInternal(ctx, job); persistUpdateAllJobErr != nil {
- slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistUpdateAllJobErr)
+ var input updateAllInput
+ if payloadErr := t.Payload(&input); payloadErr != nil {
+ return nil, errors.Join(err, payloadErr)
}
-
- s.upgradeAgentInternal(ctx, env, remote.ID, job, &job.Results[idx])
-
- if saveRemoteProgressErr := s.persistUpdateAllJobInternal(ctx, job); saveRemoteProgressErr != nil {
- slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", saveRemoteProgressErr)
+ if job, loadErr := s.loadUpdateAllJob(ctx, input.JobID); loadErr == nil {
+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("failed to list remote environments: %v", err))
}
+ return nil, err
}
-
- // All remote agents processed. Handle the manager LAST: flip its row pending ->
- // updating and move the job to pending_restart, persisting BEFORE the trigger so
- // that if the manager dies the instant the upgrader starts, the next boot sees
- // pending_restart and finalizes it.
- manager := managerResultInternal(job)
- if manager != nil {
- manager.Status = EnvironmentUpdateResultStatusUpdating
- manager.Stage = EnvironmentUpdateStageStarting
- manager.StageStartedAt = new(time.Now())
- }
- job.Status = EnvironmentUpdateJobStatusPendingRestart
- if savePendingRestartErr := s.persistUpdateAllJobInternal(ctx, job); savePendingRestartErr != nil {
- slog.WarnContext(ctx, "update-all: failed to persist pending_restart before manager upgrade", "jobId", job.ID, "error", savePendingRestartErr)
+ agents := make([]remoteAgent, len(envs))
+ for i, remote := range envs {
+ agents[i] = remoteAgent{ID: remote.ID, Name: remote.Name}
}
+ return agents, nil
+}
- // Snapshot whether the manager was already on the newest image BEFORE triggering:
- // only that makes a no-op upgrader run an expected outcome. Checked after the fact
- // the same answer proves nothing, since it was already true going in.
- wasAlreadyNewest := s.AlreadyOnNewestImage(ctx)
-
- upgraderID, err := s.TriggerUpgradeViaCLI(ctx, user, updater.SelfUpdateTarget{})
- if err != nil {
- // Agents already ran and no restart is coming, so finalize now. This flips
- // the manager's updating row to failed with the reason.
- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))
- return
+// upgradeAgentTask triggers and confirms one remote environment's upgrade, persisting each stage for the status endpoint.
+// A redelivered task skips an agent whose result is already terminal.
+func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err error) {
+ var input updateAllInput
+ var remote remoteAgent
+ if decodeErr := errors.Join(t.Payload(&input), t.DecodeItem(&remote)); decodeErr != nil {
+ return nil, decodeErr
}
-
- if manager != nil {
- s.setUpdateStageInternal(ctx, job, manager, EnvironmentUpdateStageReconnecting)
+ job, err := s.loadUpdateAllJob(ctx, input.JobID)
+ if err != nil {
+ return nil, err
}
-
- slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)
- s.watchManagerUpgraderInternal(ctx, job.ID, upgraderID, wasAlreadyNewest)
-}
-
-// watchManagerUpgraderInternal closes out a job whose manager never restarted. The
-// upgrader skips the container recreate when the pull lands on the image already
-// running, so this process survives it — and the job, persisted pending_restart for
-// the next boot to finalize, would otherwise wait for a restart that never comes.
-//
-// Runs at the tail of the agents-phase goroutine. When the manager IS recreated this
-// process is stopped instead and ResumeUpdateAllOnStartup finalizes on the next boot.
-// wasAlreadyNewest is the pre-trigger version check, the only evidence that a run
-// which changed nothing was supposed to change nothing.
-func (s *Service) watchManagerUpgraderInternal(ctx context.Context, jobID, upgraderID string, wasAlreadyNewest bool) {
- exit, exitCode := s.waitForUpgraderExitInternal(ctx, upgraderID)
- if exit == upgraderExitUnobservedInternal {
- // Never saw it finish, so a recreate may still be coming — but it may equally
- // never come, and pending_restart is only ever resolved at boot. Wait the run
- // out rather than returning: an unobserved exit must not strand the job.
- s.closeOutUnobservedUpgradeInternal(ctx, jobID)
- return
+ if job.Status != EnvironmentUpdateJobStatusRunning {
+ return nil, nil
}
-
- // A recreate stops this container, but not instantly — the stop carries a grace
- // period during which this goroutine still runs. Wait it out so a restart that is
- // already underway wins: if the manager is being replaced, the process dies here.
- time.Sleep(updateAllManagerNoRestartGraceInternal)
-
- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)
- if err != nil || job == nil {
- slog.WarnContext(ctx, "update-all: failed to reload job after upgrader exit", "jobId", jobID, "error", err)
- return
+ if allowed, authorizeErr := s.authorizeUpdateAll(ctx, job, input.KeyID); !allowed {
+ return nil, authorizeErr
}
- if job.Status != EnvironmentUpdateJobStatusPendingRestart {
- return
+ result := &job.Results[upsertPendingResult(job, remote.ID, remote.Name)]
+ if result.Status != EnvironmentUpdateResultStatusPending && result.Status != EnvironmentUpdateResultStatusUpdating {
+ return nil, nil
}
-
- if exit == upgraderExitFailedInternal {
- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade failed: upgrader exited with code %d", exitCode))
- return
+ // A saved starting or later stage means the trigger may have reached the agent; sending it again could start a second upgrader.
+ // Only a later stage proves the agent accepted it.
+ triggered := result.Status == EnvironmentUpdateResultStatusUpdating && result.Stage != "" && result.Stage != EnvironmentUpdateStageChecking
+ unconfirmed := triggered && result.Stage == EnvironmentUpdateStageStarting
+ result.Status = EnvironmentUpdateResultStatusUpdating
+ if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {
+ slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)
}
+ defer func() {
+ result.clearStage()
+ // Shutdown: leave the row updating at its stage so the redelivered task resumes it.
+ if err = ctx.Err(); err != nil {
+ return
+ }
+ if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {
+ slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)
+ }
+ }()
- info := s.versionService.GetAppVersionInfo(ctx)
+ var info versiontypes.Info
+ if triggered {
+ info.CurrentVersion, info.CurrentDigest = result.FromVersion, result.FromDigest
+ } else {
+ s.setUpdateStage(ctx, job, result, EnvironmentUpdateStageChecking)
+ versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)
+ checkErr := s.environments.ProxyJSONRequest(versionCtx, remote.ID, http.MethodGet, "/api/app-version", nil, &info)
+ cancel()
+ if checkErr != nil {
+ msg := checkErr.Error()
+ result.Status, result.Error = updateAllAgentFailureStatus(checkErr), msg[:min(len(msg), updateAllErrorMaxLen)]
+ return nil, nil
+ }
+ // The manager's newest release outranks the agent's own check, so the recorded target tracks what was sent.
+ var triggerBody []byte
+ if newest := strings.TrimSpace(s.versionService.GetAppVersionInfo(ctx).NewestVersion); newest != "" {
+ body, marshalErr := json.Marshal(TriggerUpgradeBody{TargetVersion: newest})
+ if marshalErr != nil {
+ slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", remote.ID, "error", marshalErr)
+ } else {
+ info.NewestVersion, triggerBody = newest, body
+ }
+ }
+ result.FromVersion, result.FromDigest = info.CurrentVersion, info.CurrentDigest
+ result.ToVersion = cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest)
+
+ // The saved starting stage keeps a replay from sending the trigger again, so the trigger waits on it.
+ result.Stage, result.StageStartedAt = EnvironmentUpdateStageStarting, new(time.Now())
+ if saveErr := s.db.WithContext(ctx).Save(job).Error; saveErr != nil {
+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error()
+ return nil, nil
+ }
+ triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)
+ resp, triggerErr := s.environments.ExecuteRemoteRequest(triggerCtx, remote.ID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)
+ cancel()
+ if triggerErr == nil {
+ triggerErr = resp.RequireSuccess()
+ }
+ if triggerErr != nil {
+ msg := triggerErr.Error()
+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, msg[:min(len(msg), updateAllErrorMaxLen)]
+ return nil, nil
+ }
- // Still running here means the container was never recreated, so the upgrader either
- // found the image already current or died before it got that far. With no exit code
- // to tell those apart, a no-op is only credible when the pre-trigger check already
- // said there was nothing to swap in and the post-run check still agrees: an after-
- // the-fact "already newest" on its own restates what was true before the run, so it
- // cannot vouch for it — and on a manager that was due an update it would launder a
- // dead upgrader (stale newest-image state, a check that regressed to the running
- // image) into a green row and zero logged failures, leaving a broken upgrade with no
- // retry path. Anything less is a failure; the job closes either way, since no restart
- // is coming.
- if exit == upgraderExitCodeLostInternal && (!wasAlreadyNewest || !info.AlreadyOnNewest()) {
- s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be confirmed: the upgrader exited without a readable status and this manager was not a confirmed no-op upgrade")
- return
+ // An agent already on the target finds nothing to swap in; confirming would only burn the poll window.
+ if info.AlreadyOnNewest() {
+ result.Status, result.ToVersion = EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion
+ return nil, nil
+ }
}
- s.recordManagerResultInternal(job, EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion)
- slog.InfoContext(ctx, "Update-all: manager was already up to date; no restart needed", "jobId", job.ID, "version", info.CurrentVersion)
- s.finalizeUpdateAllJobInternal(ctx, job)
-}
-
-// closeOutUnobservedUpgradeInternal fails a job whose upgrader outcome was never
-// observed and whose manager then never restarted. Waiting is what makes the answer
-// safe: a recreate kills this process, so still being here once the job would be
-// considered stale proves no restart is coming. Nothing else would close it — only a
-// boot resolves pending_restart — and until it closes, every later update-all is
-// refused as already in progress.
-func (s *Service) closeOutUnobservedUpgradeInternal(ctx context.Context, jobID string) {
- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)
- if err != nil || job == nil {
- slog.WarnContext(ctx, "update-all: failed to reload job after unobserved upgrader exit", "jobId", jobID, "error", err)
- return
- }
- if job.Status != EnvironmentUpdateJobStatusPendingRestart {
- return
+ // Poll until the agent's version lands; past the window the upgrade is only recorded as triggered.
+ // An unconfirmed request keeps its starting stage, so another restart still knows nothing confirmed it.
+ stage := func(next EnvironmentUpdateStage) {
+ if !unconfirmed {
+ s.setUpdateStage(ctx, job, result, next)
+ }
}
-
- // Hold until the job hits the same staleness bound a resumed job is judged by, so a
- // slow upgrader still gets its full window to recreate this container.
- if remaining := time.Until(job.CreatedAt.Add(updateAllStaleThresholdInternal)); remaining > 0 {
+ stage(EnvironmentUpdateStageReconnecting)
+ ticker := time.NewTicker(updateAllConfirmPollInterval)
+ defer ticker.Stop()
+ for deadline := time.Now().Add(updateAllConfirmTimeout); time.Now().Before(deadline); {
select {
case <-ctx.Done():
- return
- case <-time.After(remaining):
+ return nil, nil
+ case <-ticker.C:
+ }
+ pollCtx, pollCancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)
+ var current versiontypes.Info
+ pollErr := s.environments.ProxyJSONRequest(pollCtx, remote.ID, http.MethodGet, "/api/app-version", nil, &current)
+ pollCancel()
+ if pollErr != nil {
+ stage(EnvironmentUpdateStageReconnecting)
+ continue
+ }
+ stage(EnvironmentUpdateStageVerifying)
+ // An unconfirmed request proves nothing by the agent merely matching the target, so it needs a real change.
+ if upgradeLanded(info.CurrentVersion, info.CurrentDigest, kit.Ternary(unconfirmed, "", result.ToVersion), current.CurrentVersion, current.CurrentDigest) {
+ result.Status = EnvironmentUpdateResultStatusUpdated
+ return nil, nil
}
}
-
- job, err = s.getUpdateAllJobByIDInternal(ctx, jobID)
- if err != nil || job == nil {
- slog.WarnContext(ctx, "update-all: failed to reload stale job", "jobId", jobID, "error", err)
- return
- }
- if job.Status != EnvironmentUpdateJobStatusPendingRestart {
- return
+ if unconfirmed {
+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it"
+ return nil, nil
}
-
- slog.WarnContext(ctx, "update-all: manager upgrade was never observed and no restart followed", "jobId", job.ID)
- s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be observed: the upgrader run was never seen to finish and the manager did not restart")
+ result.Status = EnvironmentUpdateResultStatusTriggered
+ return nil, nil
}
-// upgraderExitInternal is how much the watcher managed to learn about an upgrader run.
-type upgraderExitInternal int
-
-// waitForUpgraderExitInternal blocks until the upgrader container stops, reporting what
-// could be learned about how it ended along with its exit code when one was observed.
-func (s *Service) waitForUpgraderExitInternal(ctx context.Context, upgraderID string) (upgraderExitInternal, int64) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+// upgradeManagerTask triggers the manager's own self-upgrade last. The job is persisted pending_restart BEFORE the
+// trigger, so a redelivery never triggers twice and the next boot finalizes it.
+func (s *Service) upgradeManagerTask(ctx context.Context, t flow.Task) (any, error) {
+ var input updateAllInput
+ if err := t.Payload(&input); err != nil {
+ return nil, err
+ }
+ job, err := s.loadUpdateAllJob(ctx, input.JobID)
if err != nil {
- slog.WarnContext(ctx, "update-all: cannot watch upgrader container", "upgraderId", upgraderID, "error", err)
- return upgraderExitUnobservedInternal, 0
+ return nil, err
}
-
- waitCtx, cancel := context.WithTimeout(ctx, updateAllManagerWatchTimeoutInternal)
- defer cancel()
-
- wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{
- Condition: container.WaitConditionNotRunning,
- })
- select {
- case result, ok := <-wait.Result:
- // A closed channel hands back a zero-value response, which is indistinguishable
- // from a clean exit 0 — nothing was delivered, so nothing may be concluded.
- if !ok {
- slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)
- return upgraderExitUnobservedInternal, 0
+ allowed := false
+ if job.Status == EnvironmentUpdateJobStatusRunning {
+ if allowed, err = s.authorizeUpdateAll(ctx, job, input.KeyID); err != nil {
+ return nil, err
}
- if result.StatusCode != 0 {
- return upgraderExitFailedInternal, result.StatusCode
+ }
+ if allowed {
+ // Every agent task has finished, so an agent row still in progress belongs to a task that failed first.
+ for i := range job.Results {
+ pending := job.Results[i].Status == EnvironmentUpdateResultStatusPending || job.Results[i].Status == EnvironmentUpdateResultStatusUpdating
+ if job.Results[i].EnvironmentID != environment.LocalEnvironmentID && pending {
+ job.Results[i].clearStage()
+ job.Results[i].Status, job.Results[i].Error = EnvironmentUpdateResultStatusFailed, "Agent upgrade did not finish"
+ }
}
- // The daemon fills in status code 0 alongside a wait error (the container was
- // removed mid-wait, the wait itself broke): the exit is real but its code is not.
- if result.Error != nil {
- slog.WarnContext(ctx, "update-all: upgrader wait reported an error", "upgraderId", upgraderID, "error", result.Error.Message)
- return upgraderExitCodeLostInternal, 0
+ manager := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })
+ if manager >= 0 {
+ job.Results[manager].Status = EnvironmentUpdateResultStatusUpdating
+ job.Results[manager].Stage = EnvironmentUpdateStageStarting
+ job.Results[manager].StageStartedAt = new(time.Now())
}
- return upgraderExitSucceededInternal, 0
- case waitUpgraderErr, ok := <-wait.Error:
- // The upgrader runs with AutoRemove, so a fast run can be gone before the wait
- // is acknowledged. It exited, but the code went with it — a failed upgrade must
- // not be mistaken for a clean no-op, so report the ambiguity rather than a code.
- if errdefs.IsNotFound(waitUpgraderErr) {
- return upgraderExitCodeLostInternal, 0
+ job.Status = EnvironmentUpdateJobStatusPendingRestart
+ // A replay that misses pending_restart would trigger the manager upgrade again, so the trigger waits on it.
+ if savePendingRestartErr := s.db.WithContext(ctx).Save(job).Error; savePendingRestartErr != nil {
+ return nil, fmt.Errorf("persist pending_restart before manager upgrade: %w", savePendingRestartErr)
}
- if !ok || waitUpgraderErr == nil {
- slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)
+ // Only a manager already on the newest image makes a no-op upgrader run expected; checked after the fact it proves nothing.
+ wasAlreadyNewest := s.AlreadyOnNewestImage(ctx)
+ upgraderID, triggerErr := s.TriggerUpgradeViaCLI(ctx, usertypes.Actor{ID: job.UserID, Username: job.Username}, updater.SelfUpdateTarget{})
+ if triggerErr != nil {
+ // No restart is coming, so this flips the manager's updating row to failed now.
+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", triggerErr))
} else {
- slog.WarnContext(ctx, "update-all: failed waiting for upgrader container", "upgraderId", upgraderID, "error", waitUpgraderErr)
+ if manager >= 0 {
+ s.setUpdateStage(ctx, job, &job.Results[manager], EnvironmentUpdateStageReconnecting)
+ }
+ slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)
+ s.watchManagerUpgrader(ctx, job, upgraderID, wasAlreadyNewest)
+ }
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ if job, err = s.loadUpdateAllJob(ctx, input.JobID); err != nil {
+ return nil, err
}
- return upgraderExitUnobservedInternal, 0
- case <-waitCtx.Done():
- slog.WarnContext(ctx, "update-all: timed out waiting for upgrader container", "upgraderId", upgraderID)
- return upgraderExitUnobservedInternal, 0
- }
-}
-
-// seedRemoteResultsInternal builds a pending result row for every remote environment
-// so the dialog can render the whole fleet immediately. Best effort: on error it
-// returns nil and the agents phase appends rows as it processes them.
-func (s *Service) seedRemoteResultsInternal(ctx context.Context, env *environment.EnvironmentService) EnvironmentUpdateResults {
- envs, err := env.ListRemoteEnvironments(ctx)
- if err != nil {
- slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", err)
- return nil
- }
- results := make(EnvironmentUpdateResults, 0, len(envs))
- for _, remote := range envs {
- results = append(results, EnvironmentUpdateResult{
- EnvironmentID: remote.ID,
- EnvironmentName: remote.Name,
- Status: EnvironmentUpdateResultStatusPending,
- })
- }
- return results
-}
-
-// upsertPendingResultInternal returns the index of the existing result row for envID,
-// appending a new pending row when seeding missed it (e.g. the seed list failed, or a
-// new environment was registered after the job started).
-func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName string) int {
- if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {
- return idx
}
- job.Results = append(job.Results, EnvironmentUpdateResult{
- EnvironmentID: envID,
- EnvironmentName: envName,
- Status: EnvironmentUpdateResultStatusPending,
- })
- return len(job.Results) - 1
-}
-
-// managerResultInternal returns the manager's (env "0") row, or nil when the job has none.
-func managerResultInternal(job *EnvironmentUpdateJob) *EnvironmentUpdateResult {
- idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })
- if idx < 0 {
- return nil
+ if job.Status == EnvironmentUpdateJobStatusFailed {
+ return scheduler.Outcome{Status: scheduler.Failed, Message: kit.FromPtr(job.Error)}, nil
}
- return &job.Results[idx]
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Update-all finished; " + string(job.Status)}, nil
}
-// upgradeAgentInternal triggers and confirms a single remote environment's
-// self-upgrade, recording the outcome on result. The upgrade always runs — the
-// agent pulls the latest image even when it reports no update available.
-func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult) {
- defer result.clearStageInternal()
- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageChecking)
- versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)
- var info versiontypes.Info
- err := env.ProxyJSONRequest(versionCtx, envID, http.MethodGet, "/api/app-version", nil, &info)
- cancel()
+// watchManagerUpgrader closes out a pending_restart job whose manager was never recreated, which a recreate would have
+// prevented by stopping this process. wasAlreadyNewest is the pre-trigger check that a no-op run was expected.
+func (s *Service) watchManagerUpgrader(ctx context.Context, job *EnvironmentUpdateJob, upgraderID string, wasAlreadyNewest bool) {
+ exit, exitCode := upgraderExitUnobserved, int64(0)
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- result.Status = updateAllAgentFailureStatusInternal(err)
- result.Error = truncateUpdateAllErrorInternal(err)
- return
- }
- // The manager's newest release outranks the agent's own check, mirrored into info
- // so the recorded target, up-to-date check and confirm poll track what was sent.
- var triggerBody []byte
- if managerInfo := s.versionService.GetAppVersionInfo(ctx); managerInfo != nil {
- if newest := strings.TrimSpace(managerInfo.NewestVersion); newest != "" {
- body, marshalErr := json.Marshal(TriggerUpgradeBody{TargetVersion: newest})
- if marshalErr != nil {
- slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", envID, "error", marshalErr)
+ slog.WarnContext(ctx, "update-all: cannot watch upgrader container", "upgraderId", upgraderID, "error", err)
+ } else {
+ waitCtx, cancel := context.WithTimeout(ctx, updateAllManagerWatchTimeout)
+ wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
+ select {
+ case result, ok := <-wait.Result:
+ // A closed channel yields a zero response that looks like exit 0; a wait error comes with a filler code 0.
+ switch {
+ case !ok:
+ slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)
+ case result.StatusCode != 0:
+ exit, exitCode = upgraderExitFailed, result.StatusCode
+ case result.Error != nil:
+ slog.WarnContext(ctx, "update-all: upgrader wait reported an error", "upgraderId", upgraderID, "error", result.Error.Message)
+ exit = upgraderExitCodeLost
+ default:
+ exit = upgraderExitSucceeded
+ }
+ case waitErr := <-wait.Error:
+ // AutoRemove can delete a fast upgrader before the wait lands: it exited, but its code went with it.
+ if errdefs.IsNotFound(waitErr) {
+ exit = upgraderExitCodeLost
} else {
- info.NewestVersion = newest
- triggerBody = body
+ slog.WarnContext(ctx, "update-all: failed waiting for upgrader container", "upgraderId", upgraderID, "error", waitErr)
}
+ case <-waitCtx.Done():
+ slog.WarnContext(ctx, "update-all: timed out waiting for upgrader container", "upgraderId", upgraderID)
}
+ cancel()
}
- result.FromVersion = info.CurrentVersion
- result.ToVersion = updateAllTargetVersionInternal(&info)
- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageStarting)
- triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)
- resp, err := env.ExecuteRemoteRequest(triggerCtx, envID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)
- cancel()
+ // Let a recreate already underway stop this process; an unobserved run waits out the stale window instead.
+ hold := updateAllManagerNoRestartGrace
+ if exit == upgraderExitUnobserved {
+ hold = time.Until(job.CreatedAt.Add(updateAllStaleThreshold))
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(hold):
+ }
+
+ job, err = s.loadUpdateAllJob(ctx, job.ID)
if err != nil {
- result.Status = EnvironmentUpdateResultStatusFailed
- result.Error = truncateUpdateAllErrorInternal(err)
+ slog.WarnContext(ctx, "update-all: failed to reload job after upgrader exit", "upgraderId", upgraderID, "error", err)
return
}
- if requireSuccessErr := resp.RequireSuccess(); requireSuccessErr != nil {
- result.Status = EnvironmentUpdateResultStatusFailed
- result.Error = truncateUpdateAllErrorInternal(requireSuccessErr)
+ if job.Status != EnvironmentUpdateJobStatusPendingRestart {
return
}
- // An agent that reports no update available and already runs the target has
- // nothing to swap in: its upgrader pulls, finds the same image and skips the
- // recreate. Confirming that would only burn the poll window waiting for a version
- // change that cannot come, so record it directly.
- if info.AlreadyOnNewest() {
- result.Status = EnvironmentUpdateResultStatusUpToDate
- result.ToVersion = info.CurrentVersion
- return
+ info := s.versionService.GetAppVersionInfo(ctx)
+ switch {
+ case exit == upgraderExitUnobserved:
+ slog.WarnContext(ctx, "update-all: manager upgrade was never observed and no restart followed", "jobId", job.ID)
+ s.markUpdateAllFailed(ctx, job, "manager upgrade could not be observed: the upgrader run was never seen to finish and the manager did not restart")
+ case exit == upgraderExitFailed:
+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("manager upgrade failed: upgrader exited with code %d", exitCode))
+ case exit == upgraderExitCodeLost && (!wasAlreadyNewest || !info.AlreadyOnNewest()):
+ // Without an exit code a no-op is only credible when the check said so both before and after the run.
+ s.markUpdateAllFailed(ctx, job, "manager upgrade could not be confirmed: the upgrader exited without a readable status and this manager was not a confirmed no-op upgrade")
+ default:
+ slog.InfoContext(ctx, "Update-all: manager was already up to date; no restart needed", "jobId", job.ID, "version", info.CurrentVersion)
+ s.finalizeUpdateAllJob(ctx, job, EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion)
}
+}
- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)
- if s.confirmAgentUpgradedInternal(ctx, env, envID, info, job, result) {
- result.Status = EnvironmentUpdateResultStatusUpdated
- } else {
- // Upgrade fired but the new version was not confirmed within the wait window.
- result.Status = EnvironmentUpdateResultStatusTriggered
+// upsertPendingResult returns the index of envID's result row, appending a pending row when seeding missed it.
+func upsertPendingResult(job *EnvironmentUpdateJob, envID, envName string) int {
+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {
+ return idx
}
+ job.Results = append(job.Results, EnvironmentUpdateResult{
+ EnvironmentID: envID,
+ EnvironmentName: envName,
+ Status: EnvironmentUpdateResultStatusPending,
+ })
+ return len(job.Results) - 1
}
-// updateAllAgentFailureStatusInternal classifies a failed agent pre-check. An
-// environment we actually reached but whose request failed or timed out is a real
-// failure, not an offline skip: poll-mode agents connect on demand, so a slow
-// tunnel round-trip surfaces here as a deadline even though the agent is online.
-// Only errors that look like the environment was never reachable (no tunnel,
-// connection refused, DNS failure, …) stay an offline skip.
-func updateAllAgentFailureStatusInternal(err error) EnvironmentUpdateResultStatus {
- // The tunnel/connection was established but the request did not finish: it
- // either timed out or was canceled (e.g. the parent context was aborted).
+// updateAllAgentFailureStatus classifies a failed agent pre-check: a reached environment whose request failed or timed
+// out (poll-mode tunnels connect on demand) is a failure; only a never-reachable one is an offline skip.
+func updateAllAgentFailureStatus(err error) EnvironmentUpdateResultStatus {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
return EnvironmentUpdateResultStatusFailed
}
- // The environment answered with a non-success status — reached, not offline.
if _, ok := errors.AsType[*remenv.StatusError](err); ok {
return EnvironmentUpdateResultStatusFailed
}
return EnvironmentUpdateResultStatusSkippedOffline
}
-// confirmAgentUpgradedInternal polls the agent's version until it moves off the
-// pre-upgrade baseline or reports the upgrade target, or the wait window elapses.
-// The target is the same fallback-resolved identifier recorded in ToVersion, so a
-// force-update of an already-latest agent — including one whose version check
-// could not determine the latest release — confirms on a same-image recreation
-// instead of timing out to triggered.
-func (s *Service) confirmAgentUpgradedInternal(
- ctx context.Context, env *environment.EnvironmentService, envID string,
- baseline versiontypes.Info, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult,
-) bool {
- target := updateAllTargetVersionInternal(&baseline)
- deadline := time.Now().Add(updateAllConfirmTimeoutInternal)
- ticker := time.NewTicker(updateAllConfirmPollIntervalInternal)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return false
- case <-ticker.C:
- reqCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)
- var info versiontypes.Info
- err := env.ProxyJSONRequest(reqCtx, envID, http.MethodGet, "/api/app-version", nil, &info)
- cancel()
- if err == nil {
- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageVerifying)
- versionChanged := baseline.CurrentVersion != "" && info.CurrentVersion != baseline.CurrentVersion
- digestChanged := baseline.CurrentDigest != "" && info.CurrentDigest != baseline.CurrentDigest
- onTarget := target != "" &&
- (strings.TrimPrefix(info.CurrentVersion, "v") == strings.TrimPrefix(target, "v") ||
- info.CurrentDigest == target)
- if versionChanged || digestChanged || onTarget {
- return true
- }
- } else {
- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)
- }
- if time.Now().After(deadline) {
- return false
- }
- }
- }
-}
-
-// setUpdateStageInternal records which step result is in and persists the job so
-// the status endpoint reflects it; re-entering the current stage keeps its start time.
-func (s *Service) setUpdateStageInternal(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {
+// setUpdateStage records result's step and persists the job; re-entering the current stage keeps its start time.
+func (s *Service) setUpdateStage(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {
if result.Stage == stage {
return
}
result.Stage = stage
result.StageStartedAt = new(time.Now())
- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)
}
}
// GetLatestUpdateAllJob returns the most recently created update-all job, or nil.
func (s *Service) GetLatestUpdateAllJob(ctx context.Context) (*EnvironmentUpdateJob, error) {
- var jobs []EnvironmentUpdateJob
- if err := s.db.WithContext(ctx).
- Order("created_at DESC").
- Limit(1).
- Find(&jobs).Error; err != nil {
- return nil, err
- }
- if len(jobs) == 0 {
- return nil, nil
- }
- return &jobs[0], nil
+ return s.findUpdateAllJob(ctx)
}
-func (s *Service) activeUpdateAllJobInternal(ctx context.Context) (*EnvironmentUpdateJob, error) {
+// findUpdateAllJob returns the newest job matching the inline GORM conditions, or nil when none does.
+func (s *Service) findUpdateAllJob(ctx context.Context, conds ...any) (*EnvironmentUpdateJob, error) {
var jobs []EnvironmentUpdateJob
- if err := s.db.WithContext(ctx).
- Where("status IN ?", []string{
- string(EnvironmentUpdateJobStatusPendingRestart),
- string(EnvironmentUpdateJobStatusRunning),
- }).
- Order("created_at DESC").
- Limit(1).
- Find(&jobs).Error; err != nil {
+ if err := s.db.WithContext(ctx).Order("created_at DESC").Limit(1).Find(&jobs, conds...).Error; err != nil {
return nil, err
}
if len(jobs) == 0 {
@@ -1243,42 +1011,46 @@ func (s *Service) activeUpdateAllJobInternal(ctx context.Context) (*EnvironmentU
return &jobs[0], nil
}
-func (s *Service) getUpdateAllJobByIDInternal(ctx context.Context, id string) (*EnvironmentUpdateJob, error) {
- var jobs []EnvironmentUpdateJob
- if err := s.db.WithContext(ctx).
- Where("id = ?", id).
- Limit(1).
- Find(&jobs).Error; err != nil {
- return nil, err
- }
- if len(jobs) == 0 {
- return nil, nil
+// loadUpdateAllJob loads a job by ID, failing when it no longer exists.
+func (s *Service) loadUpdateAllJob(ctx context.Context, id string) (*EnvironmentUpdateJob, error) {
+ job, err := s.findUpdateAllJob(ctx, "id = ?", id)
+ if err == nil && job == nil {
+ err = fmt.Errorf("update-all job %s not found", id)
}
- return &jobs[0], nil
+ return job, err
}
-func (s *Service) persistUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) error {
- return s.db.WithContext(ctx).Save(job).Error
+// authorizeUpdateAll reports whether the job's requester may still upgrade, failing the job when not.
+// A task recovered after a restart never passed the request's permission check. Shutdown returns its
+// error so the task resumes instead of failing the job.
+func (s *Service) authorizeUpdateAll(ctx context.Context, job *EnvironmentUpdateJob, keyID string) (bool, error) {
+ permissions, err := s.roles.ResolveExecutionPermissions(ctx, job.UserID, keyID)
+ if ctx.Err() != nil {
+ return false, ctx.Err()
+ }
+ if err == nil && permissions.Allows(authz.PermSystemUpgrade, environment.LocalEnvironmentID) {
+ return true, nil
+ }
+ s.markUpdateAllFailed(ctx, job, "requesting user no longer has permission to upgrade")
+ return false, nil
}
-func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *EnvironmentUpdateJob, reason string) {
+func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdateJob, reason string) {
job.Status = EnvironmentUpdateJobStatusFailed
job.Error = &reason
job.CompletedAt = new(time.Now())
for i := range job.Results {
- job.Results[i].clearStageInternal()
+ job.Results[i].clearStage()
if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {
job.Results[i].Status = EnvironmentUpdateResultStatusFailed
job.Results[i].Error = reason
}
}
- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to mark job failed", "jobId", job.ID, "error", err)
}
- // Surface the failure in the events audit log; the success path logs via
- // logUpdateAllEventInternal. LogUserEvent hardcodes an info-severity "completed"
- // title, so create the event directly with error severity and the reason.
+ // LogUserEvent always records an info-severity "completed" event, so create the error event directly.
if _, err := s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeSystemUpgrade,
Severity: event.EventSeverityError,
@@ -1299,51 +1071,29 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environm
slog.WarnContext(ctx, "Update-all job failed", "jobId", job.ID, "reason", reason)
}
-// recordManagerResultInternal sets the manager (env "0") entry to its final status:
-// updated after a confirmed restart, up_to_date when the pull found nothing to swap
-// in, or failed otherwise.
-func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, status EnvironmentUpdateResultStatus, currentVersion string) {
- manager := managerResultInternal(job)
- if manager == nil {
- return
- }
- manager.Status = status
- switch status {
- case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:
- manager.ToVersion = currentVersion
- case EnvironmentUpdateResultStatusFailed:
- manager.Error = "manager version did not change after upgrade"
- case EnvironmentUpdateResultStatusPending,
- EnvironmentUpdateResultStatusUpdating,
- EnvironmentUpdateResultStatusTriggered,
- EnvironmentUpdateResultStatusSkippedOffline:
- // Nothing more to record: the row keeps the target version it was seeded
- // with, since none of these outcomes establishes what it ended up running.
+// finalizeUpdateAllJob settles the manager row, completes the job and records the audit event.
+func (s *Service) finalizeUpdateAllJob(ctx context.Context, job *EnvironmentUpdateJob, managerStatus EnvironmentUpdateResultStatus, managerVersion string) {
+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID }); idx >= 0 {
+ job.Results[idx].Status = managerStatus
+ if managerStatus == EnvironmentUpdateResultStatusFailed {
+ job.Results[idx].Error = "manager version did not change after upgrade"
+ } else {
+ job.Results[idx].ToVersion = managerVersion
+ }
}
-}
-
-// finalizeUpdateAllJobInternal closes a job out as completed and records the audit
-// event. The per-environment rows must already carry their final statuses.
-func (s *Service) finalizeUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) {
job.Status = EnvironmentUpdateJobStatusCompleted
job.CompletedAt = new(time.Now())
+ failed := 0
for i := range job.Results {
- job.Results[i].clearStageInternal()
+ job.Results[i].clearStage()
+ if job.Results[i].Status == EnvironmentUpdateResultStatusFailed {
+ failed++
+ }
}
- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {
slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)
return
}
- s.logUpdateAllEventInternal(ctx, job)
-}
-
-func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *EnvironmentUpdateJob) {
- failed := 0
- for _, r := range job.Results {
- if r.Status == EnvironmentUpdateResultStatusFailed {
- failed++
- }
- }
metadata := database.JSON{
"action": "update_all_environments",
@@ -1351,17 +1101,13 @@ func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *Environmen
"environments": len(job.Results),
"failed": failed,
}
-
- // All environments succeeded: log the standard completed (info) event.
if failed == 0 {
if err := s.eventService.LogUserEvent(ctx, event.EventTypeSystemUpgrade, job.UserID, job.Username, metadata); err != nil {
slog.WarnContext(ctx, "Failed to log update-all event", "jobId", job.ID, "error", err)
}
return
}
-
- // The job ran to completion but some environments failed to update — record a
- // warning-severity event so those failures still show in the audit log.
+ // Completed with failures: a warning event keeps them visible in the audit log.
if _, err := s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeSystemUpgrade,
Severity: event.EventSeverityWarning,
@@ -1375,40 +1121,6 @@ func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *Environmen
}
}
-// updateAllTargetVersionInternal picks the best human-readable target identifier:
-// the newest version tag if known, otherwise the newest digest. When the version
-// check could not determine the latest release (offline, rate-limited) it falls
-// back to the current identifiers: updates run unconditionally, so the target of a
-// force-update with an unknown latest is wherever the pull lands — recording the
-// current version keeps the resume check able to recognize a same-image recreation
-// as success instead of finalizing it as failed.
-func updateAllTargetVersionInternal(info *versiontypes.Info) string {
- if info == nil {
- return ""
- }
- if info.NewestVersion != "" {
- return info.NewestVersion
- }
- if info.NewestDigest != "" {
- return info.NewestDigest
- }
- if info.CurrentVersion != "" {
- return info.CurrentVersion
- }
- return info.CurrentDigest
-}
-
-func truncateUpdateAllErrorInternal(err error) string {
- if err == nil {
- return ""
- }
- msg := err.Error()
- if len(msg) > updateAllErrorMaxLenInternal {
- return msg[:updateAllErrorMaxLenInternal]
- }
- return msg
-}
-
// PruneUpgradeLogs removes expired upgrade logs from this instance's data directory.
func (s *Service) PruneUpgradeLogs(ctx context.Context, dataDir string, now time.Time) (int, error) {
retentionDays := s.settingsService.GetIntSetting(ctx, "upgradeLogRetentionDays", 3)
@@ -1446,7 +1158,7 @@ func (s *Service) PruneUpgradeLogs(ctx context.Context, dataDir string, now time
if cancellationErr := ctx.Err(); cancellationErr != nil {
return removed, errors.Join(append(failures, cancellationErr)...)
}
- if !upgradeLogNameInternal.MatchString(entry.Name()) {
+ if !upgradeLogName.MatchString(entry.Name()) {
continue
}
info, lstatErr := root.Lstat(entry.Name())
diff --git a/backend/internal/system/children/upgrade/service_test.go b/backend/internal/system/children/upgrade/service_test.go
index b131734ff..9c2350cd5 100644
--- a/backend/internal/system/children/upgrade/service_test.go
+++ b/backend/internal/system/children/upgrade/service_test.go
@@ -23,6 +23,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
)
// TestService_UpgradeFlag tests the upgrading flag behavior
@@ -192,14 +193,14 @@ func TestUpdateAllAgentFailureStatus(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- require.Equal(t, tt.want, updateAllAgentFailureStatusInternal(tt.err))
+ require.Equal(t, tt.want, updateAllAgentFailureStatus(tt.err))
})
}
}
// A blank-target self-upgrade resolves its image from the version check (#3687).
// Explicit targets from the updater engine never reach this function.
-func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {
+func TestResolveSelfUpgradeTargetImage(t *testing.T) {
tests := []struct {
name string
currentImage string
@@ -259,7 +260,7 @@ func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got, err := resolveSelfUpgradeTargetImageInternal(tt.currentImage, tt.info)
+ got, err := resolveSelfUpgradeTargetImage(tt.currentImage, tt.info)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
return
@@ -342,7 +343,7 @@ func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {
Labels: map[string]string{"com.docker.compose.project": "arcane", "com.docker.compose.service": "server"},
}}
- got, saveCompose, err := svc.configuredTargetImageInternal(t.Context(), current, tt.target)
+ got, saveCompose, err := svc.configuredTargetImage(t.Context(), current, tt.target)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
require.Nil(t, saveCompose)
@@ -425,7 +426,7 @@ func TestUpdateAllResolveResumeAction(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := resolveResumeActionInternal(tt.job, tt.currentVersion, tt.currentDigest, now)
+ got := resolveResumeAction(tt.job, tt.currentVersion, tt.currentDigest, now)
require.Equal(t, tt.wantStale, got.markStale,
"markStale = %v, want %v", got.markStale, tt.wantStale)
@@ -436,15 +437,6 @@ func TestUpdateAllResolveResumeAction(t *testing.T) {
}
}
-// A force-update with an unknown latest (offline or rate-limited version check)
-// must still record a target — the current identifiers — so the resume check can
-// recognize a same-image recreation as success instead of finalizing it as failed.
-func TestUpdateAllTargetVersionFallsBackToCurrent(t *testing.T) {
- require.Equal(t, "v2.0.0", updateAllTargetVersionInternal(&versiontypes.Info{NewestVersion: "v2.0.0", CurrentVersion: "v1.2.3"}))
- require.Equal(t, "v1.2.3", updateAllTargetVersionInternal(&versiontypes.Info{CurrentVersion: "v1.2.3", CurrentDigest: "sha256:a"}))
- require.Equal(t, "sha256:a", updateAllTargetVersionInternal(&versiontypes.Info{CurrentDigest: "sha256:a"}))
-}
-
func TestUpsertPendingResult(t *testing.T) {
job := &EnvironmentUpdateJob{
Results: EnvironmentUpdateResults{
@@ -455,7 +447,7 @@ func TestUpsertPendingResult(t *testing.T) {
{
// A seeded environment resolves to its existing row without appending.
- idx := upsertPendingResultInternal(job, "abc", "palladium")
+ idx := upsertPendingResult(job, "abc", "palladium")
require.Equal(t, 1, idx,
"existing env index = %d, want 1", idx)
}
@@ -465,7 +457,7 @@ func TestUpsertPendingResult(t *testing.T) {
// A missing environment (seeding raced or a new env was registered) appends a
// fresh pending row and returns the new index.
- idx := upsertPendingResultInternal(job, "xyz", "oracle-cloud")
+ idx := upsertPendingResult(job, "xyz", "oracle-cloud")
require.Equal(t, 2, idx,
"new env index = %d, want 2", idx)
@@ -505,7 +497,7 @@ func TestUpdateAllFailedJobMarksUpdatingResultsFailed(t *testing.T) {
require.NoError(t, db.WithContext(ctx).Create(job).Error)
reason := "interrupted by manager restart"
- svc.markUpdateAllFailedInternal(ctx, job, reason)
+ svc.markUpdateAllFailed(ctx, job, reason)
var got EnvironmentUpdateJob
require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)
@@ -555,8 +547,7 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {
}
require.NoError(t, db.WithContext(ctx).Create(job).Error)
- svc.recordManagerResultInternal(job, EnvironmentUpdateResultStatusUpToDate, "v1.0.0")
- svc.finalizeUpdateAllJobInternal(ctx, job)
+ svc.finalizeUpdateAllJob(ctx, job, EnvironmentUpdateResultStatusUpToDate, "v1.0.0")
var got EnvironmentUpdateJob
require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)
@@ -594,15 +585,15 @@ func TestUpdateAllStageChangesPersist(t *testing.T) {
}
require.NoError(t, db.WithContext(ctx).Create(job).Error)
- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
started := job.Results[0].StageStartedAt
require.NotNil(t, started)
require.WithinDuration(t, time.Now(), *started, time.Second)
- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
require.Same(t, started, job.Results[0].StageStartedAt)
- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)
+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)
var got EnvironmentUpdateJob
require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)
require.Equal(t, EnvironmentUpdateStageStarting, got.Results[0].Stage)
@@ -688,6 +679,10 @@ func TestResumeUpdateAllFinalizesManagerWithoutRerunningAgents(t *testing.T) {
// is judged successful.
versionSvc := version.NewVersionService(nil, true, "v9.9.9-new", "", nil, nil, nil, nil)
svc := NewService(db, nil, versionSvc, event.NewEventService(db, nil, nil), nil, nil, nil)
+ harness := flowtest.New(t, nil)
+ require.NoError(t, svc.RegisterWorkflows(harness.Engine, nil, nil))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
job := &EnvironmentUpdateJob{
Status: EnvironmentUpdateJobStatusPendingRestart,
diff --git a/backend/internal/system/module.go b/backend/internal/system/module.go
index e5f52fdb2..27df6978c 100644
--- a/backend/internal/system/module.go
+++ b/backend/internal/system/module.go
@@ -10,7 +10,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/prune"
@@ -21,22 +20,20 @@ import (
// Module wires the system domain and mounts its routes.
type Module struct {
- service *SystemService
- dockerService *docker.DockerClientService
- environmentService *environment.EnvironmentService
- activityService *activity.ActivityService
- cfg *config.Config
+ service *SystemService
+ dockerService *docker.DockerClientService
+ activityService *activity.ActivityService
+ cfg *config.Config
}
// New wires system routes around an existing service.
func New(
service *SystemService,
dockerService *docker.DockerClientService,
- environmentService *environment.EnvironmentService,
activityService *activity.ActivityService,
cfg *config.Config,
) *Module {
- return &Module{service: service, dockerService: dockerService, environmentService: environmentService, activityService: activityService, cfg: cfg}
+ return &Module{service: service, dockerService: dockerService, activityService: activityService, cfg: cfg}
}
// Service exposes the system service to collaborators.
@@ -60,7 +57,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))
prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))
- upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.environmentService, m.cfg, appCtx.Context()))
+ upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.cfg, appCtx.Context()))
backup.RegisterRoutes(api, backup.NewHandler(service.backup, m.activityService, appCtx.Context()))
}
diff --git a/backend/internal/system/service.go b/backend/internal/system/service.go
index d8cea128a..d3519925d 100644
--- a/backend/internal/system/service.go
+++ b/backend/internal/system/service.go
@@ -37,10 +37,12 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/network"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"
@@ -50,6 +52,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
@@ -126,13 +129,22 @@ func NewSystemService(
return s
}
-// ReconcileInterruptedBackups fails system backup runs left running by the previous process.
+// RegisterWorkflows defines the backup workflows while the host is still unstarted.
+func (s *SystemService) RegisterWorkflows(engine *flow.Engine, environments *environment.EnvironmentService, roles *role.RoleService) error {
+ return errors.Join(s.backup.RegisterWorkflows(engine), s.upgrade.RegisterWorkflows(engine, environments, roles))
+}
+
+// ReconcileInterruptedBackups fails system backup runs left running by the previous process and watches the
+// manual backups that saved workflows resume.
func (s *SystemService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
query := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning)
if len(protectedIDs) > 0 {
query = query.Where("id NOT IN ?", protectedIDs)
}
- return query.Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error
+ if err := query.Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error; err != nil {
+ return err
+ }
+ return s.backup.WatchAcceptedBackups(ctx)
}
// SetBackupScheduler injects the dynamic scheduler and admission gate for
@@ -472,9 +484,9 @@ func resolveUpgraderRuntimeOptionsInternal(
}
const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger, destination, '' AS format, local_snapshot_id, " +
- "remote_snapshot_id, s3_destination_id, policy_id, error, 'system' AS type, 'system' AS " +
+ "remote_snapshot_id, '' AS remote_instance_id, s3_destination_id, policy_id, error, 'system' AS type, 'system' AS " +
"resource_type, 'Arcane' AS resource_name FROM system_backup_runs UNION ALL SELECT id, size, " +
- "created_at, status, trigger, destination, format, local_snapshot_id, remote_snapshot_id, " +
+ "created_at, status, trigger, destination, format, local_snapshot_id, remote_snapshot_id, COALESCE(remote_instance_id, '') AS remote_instance_id, " +
"s3_destination_id, policy_id, error, CASE WHEN policy_id LIKE 'system-volume:%' THEN 'system' ELSE " +
"'volume' END AS type, 'volume' AS resource_type, volume_name AS resource_name FROM volume_backups"
@@ -622,15 +634,8 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ
},
}.Run(ctx, updates)
},
- DisablePolicyRemote: func(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error) {
- column := "enabled"
- if policy.LocalEnabled {
- column = "s3_enabled"
- }
- result := s.db.WithContext(ctx).Model(&SystemBackupPolicy{}).
- Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).
- Update(column, false)
- return result.Error == nil && result.RowsAffected > 0, result.Error
+ DisablePolicyRemote: func(ctx context.Context, policy *backuptypes.SystemBackupPolicy, field string) (bool, error) {
+ return backup.DisableStoredRemote(ctx, s.db, &SystemBackupPolicy{}, policy.ID, policy.S3DestinationID, policy.LocalEnabled, field)
},
CheckScheduledRemote: func(ctx context.Context, destinationID string) error {
return backup.CheckScheduledRemote(ctx, s.db, s3Destinations, "system_backup_runs", destinationID, "arcane-system-recovery")
diff --git a/backend/internal/system/service_test.go b/backend/internal/system/service_test.go
index 440f23207..bc2799023 100644
--- a/backend/internal/system/service_test.go
+++ b/backend/internal/system/service_test.go
@@ -20,6 +20,7 @@ import (
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -27,6 +28,7 @@ import (
systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
@@ -196,7 +198,7 @@ func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request
func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}, &activity.Activity{}, &activity.ActivityMessage{}))
crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})
db := &database.DB{DB: gormDB}
recoveryKeys := backup.NewRecoveryKeyStore(db)
@@ -210,6 +212,9 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
Config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
RecoveryKeys: recoveryKeys,
})
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ require.NoError(t, service.backup.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
jobScheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}
require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, newSystemBackupAdmissionGateForTestInternal(t)))
@@ -260,10 +265,11 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
// The scheduled run checks the remote first; a local policy then
// attempts its backup, which has no engine in this test.
outcome, runErr := job.Run(t.Context())
+ require.NoError(t, runErr)
if local {
- require.ErrorContains(t, runErr, "backup engine is unavailable")
+ require.Equal(t, scheduler.Failed, outcome.Status)
+ require.Contains(t, outcome.Message, "backup engine is unavailable")
} else {
- require.NoError(t, runErr)
require.Equal(t, scheduler.NeedsAttention, outcome.Status)
require.Equal(t, backup.RemoteDisabledMessage, outcome.Message)
}
diff --git a/backend/internal/template/handler.go b/backend/internal/template/handler.go
index 073ea9d09..d4b1d5119 100644
--- a/backend/internal/template/handler.go
+++ b/backend/internal/template/handler.go
@@ -257,14 +257,7 @@ func (h *TemplateHandler) DeleteTemplate(ctx context.Context, input *DeleteTempl
return nil, huma.Error500InternalServerError("Failed to delete template: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Template deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Template deleted successfully", ""), nil
}
// DownloadTemplate downloads a remote template to local storage.
@@ -332,14 +325,7 @@ func (h *TemplateHandler) SaveDefaultTemplates(ctx context.Context, input *SaveD
return nil, huma.Error500InternalServerError("Failed to save default template: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Default templates saved successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Default templates saved successfully", ""), nil
}
// GetRegistries returns all template registries.
@@ -416,14 +402,7 @@ func (h *TemplateHandler) UpdateRegistry(ctx context.Context, input *UpdateTempl
return nil, huma.Error500InternalServerError("Failed to update registry: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Registry updated successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Registry updated successfully", ""), nil
}
// DeleteRegistry deletes a template registry.
@@ -439,14 +418,7 @@ func (h *TemplateHandler) DeleteRegistry(ctx context.Context, input *DeleteTempl
return nil, huma.Error500InternalServerError("Failed to delete registry: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Registry deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Registry deleted successfully", ""), nil
}
// FetchRegistry fetches templates from a remote registry URL.
diff --git a/backend/internal/updater/children/execution/service.go b/backend/internal/updater/children/execution/service.go
index b803368bb..5573dadcf 100644
--- a/backend/internal/updater/children/execution/service.go
+++ b/backend/internal/updater/children/execution/service.go
@@ -2,493 +2,177 @@ package execution
import (
"context"
+ "encoding/json/v2"
"errors"
- "fmt"
- "log/slog"
- "sync"
+ "slices"
"time"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/updater"
"github.com/italypaleale/francis/actor"
- "github.com/italypaleale/francis/host/local"
- "gorm.io/gorm"
+ "github.com/italypaleale/francis/builtin/workflow"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
)
-// Dependencies are the updater operations single-container delivery runs on.
-type Dependencies struct {
- Coordinator *runs.Coordinator
- Config *config.Config
- Roles *role.RoleService
- Activity *activity.ActivityService
- Logger func() *slog.Logger
- AcquireUpdate func(ctx context.Context) (context.Context, func(), error)
- UpdateBusy error
- TrackActivity func(ctx context.Context, activityID string) context.Context
- RunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)
- FinishUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error)
- FreezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)
- ConfirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)
- WithFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context
-}
-
-// Service delivers accepted single-container updates through durable Francis
-// jobs and repairs interrupted deliveries.
-type Service struct {
- coordinator *runs.Coordinator
- config *config.Config
- roles *role.RoleService
- activity *activity.ActivityService
- logger func() *slog.Logger
- acquireUpdate func(ctx context.Context) (context.Context, func(), error)
- updateBusy error
- trackActivity func(ctx context.Context, activityID string) context.Context
- runUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)
- finishUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error)
- freezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)
- confirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)
- withFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context
-
- singleUpdates *actor.Service
- singleMu sync.Mutex
- singleContext context.Context
- singleCancel context.CancelFunc
- singleWorkers sync.WaitGroup
- singleDone chan struct{}
- singleStopping bool
-}
-
-func NewService(deps Dependencies) *Service {
- return &Service{
- coordinator: deps.Coordinator,
- config: deps.Config,
- roles: deps.Roles,
- activity: deps.Activity,
- logger: deps.Logger,
- acquireUpdate: deps.AcquireUpdate,
- updateBusy: deps.UpdateBusy,
- trackActivity: deps.TrackActivity,
- runUpdate: deps.RunUpdate,
- finishUpdate: deps.FinishUpdate,
- freezeContainer: deps.FreezeContainer,
- confirmTarget: deps.ConfirmTarget,
- withFrozenTarget: deps.WithFrozenTarget,
- }
-}
-
-// Ready reports whether the actor host has been registered.
-func (s *Service) Ready() bool {
- return s.singleUpdates != nil
-}
-
-// Submit persists an accepted update intent and dispatches it without waiting
-// for the update to run.
-func (s *Service) Submit(ctx, workCtx context.Context, containerID, activityID string) error {
- command := singleUpdateCommandInternal(ctx, containerID, activityID)
- state := updater.SingleUpdateState{Command: command, Status: "queued"}
- if setStateErr := s.singleUpdates.SetState(workCtx, StateType, activityID, state, nil); setStateErr != nil {
- s.finishUpdate(workCtx, activityID, nil, setStateErr)
- return fmt.Errorf("persist container update: %w", setStateErr)
+type (
+ // Dependencies are the updater operations a single-container update runs on.
+ Dependencies struct {
+ Config *config.Config
+ Roles *role.RoleService
+ AcquireUpdate func(ctx context.Context) (context.Context, func(), error)
+ UpdateBusy error
+ RunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)
+ // RecordUpdate saves the result on the activity and summarizes it as the run outcome.
+ RecordUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error) scheduler.Outcome
+ FreezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)
+ ConfirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)
+ WithFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context
+ }
+
+ // Service runs accepted single-container updates as container-update workflows.
+ Service struct {
+ deps Dependencies
+ flow *flow.Engine
+ workflow *flow.Workflow
+ }
+
+ // updateInput is an accepted update: the container and who asked for it.
+ updateInput struct {
+ ContainerID string `json:"containerId"`
+ UserID string `json:"userId,omitempty"`
+ KeyID string `json:"keyId,omitempty"`
}
- if dispatchSingleErr := s.dispatchSingleInternal(workCtx, state); dispatchSingleErr != nil {
- s.logger().WarnContext(workCtx, "container update dispatch deferred", "activityId", activityID, "error", dispatchSingleErr)
- }
- return nil
-}
-
-// ActorType and StateType are the durable Francis identifiers of
-// single-container update jobs and their persisted delivery state.
-const (
- ActorType = "single-container-update"
- StateType = "single-container-update-state"
)
-type singleUpdateActorInternal struct{ service *Service }
-
-func (s *Service) RegisterActors(runtime *francis.Runtime) error {
- s.singleUpdates = runtime.Service()
- return runtime.RegisterActor(
- ActorType,
- func(
- _ string,
- _ *actor.Service,
- ) actor.Actor {
- return &singleUpdateActorInternal{
- service: s,
- }
+func NewService(deps Dependencies) *Service {
+ return &Service{deps: deps}
+}
+
+// RegisterWorkflows defines the container-update workflow while the host is still unstarted.
+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
+ var err error
+ s.flow = engine
+ s.workflow, err = engine.Define(flow.Definition{
+ Name: "container-update",
+ Version: 1,
+ Fingerprint: "1b1bad80429a7061e5541b6fa57eb43b187046c3987bec83ea39f0496589ccf8",
+ Concurrency: 4,
+ Timeout: 2 * time.Hour,
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeAutoUpdate, EnvironmentID: "0", Queue: true, DeferSlot: true,
+ ResourceType: new("container"), Step: "Updating container", LatestMessage: "Container update started",
},
- local.WithCapacityGroup(
- "jobs",
- 4,
- ),
- local.WithCompletedJobRetention(
- 7*24*time.Hour,
- ),
- )
-}
-
-// Start repairs persisted single-container delivery intents until shutdown.
-func (s *Service) Start(ctx context.Context) error {
- s.singleMu.Lock()
- defer s.singleMu.Unlock()
- if s.singleContext != nil {
- return nil
- }
- s.singleContext, s.singleCancel = context.WithCancel(ctx)
- s.singleDone = make(chan struct{})
- go func() {
- defer close(s.singleDone)
- ticker := time.NewTicker(5 * time.Second)
- defer ticker.Stop()
- for {
- if s.coordinator == nil || s.coordinator.Active() {
- if err := s.repairSinglesInternal(s.singleContext); err != nil && s.singleContext.Err() == nil {
- s.logger().WarnContext(s.singleContext, "repair container update dispatch", "error", err)
- }
- }
- select {
- case <-s.singleContext.Done():
- return
- case <-ticker.C:
- }
- }
- }()
- return nil
+ Labels: map[string]string{"update": "Updating container"},
+ Steps: []workflow.StepSpec{workflow.Step("update", engine.Handler(s.update), workflow.WithMaxAttempts(1))},
+ })
+ return err
}
-// Stop cancels business execution before the actor host drains jobs.
-func (s *Service) Stop(ctx context.Context) error {
- s.singleMu.Lock()
- s.singleStopping = true
- if s.singleCancel != nil {
- s.singleCancel()
- }
- done := s.singleDone
- s.singleMu.Unlock()
- joined := make(chan struct{})
- go func() {
- if done != nil {
- <-done
- }
- s.singleWorkers.Wait()
- close(joined)
- }()
- select {
- case <-joined:
- return nil
- case <-ctx.Done():
- return ctx.Err()
+// Submit accepts an update of the container and returns its activity ID; activity describes that activity.
+func (s *Service) Submit(ctx context.Context, containerID string, activity activitylib.StartRequest) (string, error) {
+ input := updateInput{ContainerID: containerID}
+ if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {
+ input.UserID = user.ID
}
+ input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ return s.flow.Submit(ctx, s.workflow, input, activity)
}
-func (s *Service) dispatchSingleInternal(ctx context.Context, state updater.SingleUpdateState) error {
- if s.coordinator != nil && !s.coordinator.Active() {
- return nil
+// update is the container-update step. A delivery that finds the target it froze confirms that target's effect
+// before touching Docker again; while another update holds the updater, the task is handed back.
+func (s *Service) update(ctx context.Context, t flow.Task) (any, error) {
+ var input updateInput
+ if err := t.Payload(&input); err != nil {
+ return nil, err
}
- id, _, err := s.singleUpdates.Dispatch(ctx, ActorType, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))
- if err != nil {
- return err
+ ctx, release, err := s.deps.AcquireUpdate(ctx)
+ if errors.Is(err, s.deps.UpdateBusy) {
+ return nil, actor.ErrJobRejected
}
- job, err := s.singleUpdates.GetJob(ctx, id)
if err != nil {
- return err
+ return nil, err
}
- if !job.Status.IsTerminal() {
- return nil
+ defer release()
+ activityID := t.ActivityID()
+ failed := func(reason string) scheduler.Outcome {
+ return scheduler.Outcome{Status: scheduler.Failed, Message: reason}
}
- // A terminal transport record cannot prove the durable business outcome.
- // Its replacement reconciles the saved effect before doing any further work.
- if deleteJobErr := s.singleUpdates.DeleteJob(ctx, ActorType, "single-container", id); deleteJobErr != nil {
- return deleteJobErr
+ if input.UserID != "" && (s.deps.Config == nil || !s.deps.Config.AgentMode || input.UserID != "agent" || input.KeyID != "") {
+ if s.deps.Roles == nil {
+ return failed("requesting user permissions unavailable"), nil
+ }
+ permissions, permissionsErr := s.deps.Roles.ResolveExecutionPermissions(ctx, input.UserID, input.KeyID)
+ if permissionsErr != nil || !permissions.Allows(authz.PermImageUpdatesCheck, "0") {
+ return failed("requesting user no longer has permission to update containers"), nil
+ }
}
- _, _, err = s.singleUpdates.Dispatch(ctx, ActorType, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))
- return err
-}
-// repairSinglesInternal scans every persisted update, collecting per-entry
-// failures so one bad entry cannot block repair of the rest.
-func (s *Service) repairSinglesInternal(ctx context.Context) error {
- var failures []error
- for cursor := ""; ; {
- page, err := s.singleUpdates.ListStates(ctx, StateType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
- if err != nil {
- return errors.Join(append(failures, err)...)
+ var target updater.FrozenUpdateTarget
+ previous, _ := jobcontext.Run(ctx)
+ if index := slices.IndexFunc(previous.Outcome.Targets, func(evidence scheduler.TargetOutcome) bool {
+ return evidence.ID == input.ContainerID && len(evidence.RecoveryData) > 0
+ }); index >= 0 {
+ if decodeErr := json.Unmarshal(previous.Outcome.Targets[index].RecoveryData, &target); decodeErr != nil {
+ return nil, decodeErr
}
- for _, entry := range page.States {
- if ctxErr := ctx.Err(); ctxErr != nil {
- return errors.Join(append(failures, ctxErr)...)
- }
- state, decodeErr := decodeSingleStateInternal(entry)
- if decodeErr == nil {
- decodeErr = s.repairSingleInternal(ctx, state)
- }
- if decodeErr != nil {
- failures = append(failures, fmt.Errorf("repair container update %s: %w", entry.ActorID, decodeErr))
+ confirmed, unchanged, confirmErr := s.deps.ConfirmTarget(ctx, target)
+ switch {
+ case confirmErr != nil:
+ return nil, confirmErr
+ case confirmed:
+ item := updater.ResourceResult{
+ ResourceID: target.ContainerID, ResourceName: target.ContainerName, ResourceType: "container",
+ Status: updater.StatusUpdated, UpdateApplied: true,
}
+ return s.deps.RecordUpdate(ctx, activityID, &updater.Result{Updated: 1, Items: []updater.ResourceResult{item}}, nil), nil
+ case !unchanged:
+ return failed("Interrupted container update effect cannot be confirmed"), nil
}
- cursor = page.AfterID()
- if cursor == "" {
- return errors.Join(failures...)
+ } else {
+ frozen, freezeErr := s.deps.FreezeContainer(ctx, input.ContainerID)
+ if freezeErr != nil {
+ return failed(freezeErr.Error()), nil
}
+ target = *frozen
}
-}
-
-func (s *Service) repairSingleInternal(ctx context.Context, state updater.SingleUpdateState) error {
- if state.Status == "queued" || state.Status == "running" {
- return s.dispatchSingleInternal(ctx, state)
- }
- if s.activity == nil {
- return nil
- }
- detail, err := s.activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)
- // Pruned or deleted history needs no projection repair.
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil
- }
- if err != nil {
- return err
- }
- if detail.Activity.Status != activitytypes.StatusQueued && detail.Activity.Status != activitytypes.StatusRunning {
- return nil
- }
- var outcomeErr error
- if state.Failure != "" {
- outcomeErr = errors.New(state.Failure)
- }
- s.finishUpdate(ctx, state.Command.ActivityID, state.Result, outcomeErr)
- return nil
-}
-
-func decodeSingleStateInternal(entry actor.StateInfo) (updater.SingleUpdateState, error) {
- var state updater.SingleUpdateState
- if entry.Data == nil {
- return state, errors.New("empty update state")
- }
- err := entry.Data.Decode(&state)
- return state, err
-}
-func (a *singleUpdateActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {
- s := a.service
- if s.coordinator != nil && !s.coordinator.Active() {
- return actor.ErrJobRejected
- }
- s.singleMu.Lock()
- if s.singleStopping {
- s.singleMu.Unlock()
- return actor.ErrJobRejected
- }
- s.singleWorkers.Add(1)
- lifetime := s.singleContext
- s.singleMu.Unlock()
- defer s.singleWorkers.Done()
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
- if lifetime != nil {
- //nolint:contextcheck // The service lifetime cancels the inherited Francis job context before host drain.
- stop := context.AfterFunc(lifetime, cancel)
- defer stop()
- }
- ctx, release, err := s.acquireUpdate(ctx)
- if err != nil {
- if errors.Is(err, s.updateBusy) {
- return actor.ErrJobRejected
+ // The frozen target is the evidence a redelivery confirms, so it is recorded before and as the updater pins its image.
+ record := func() error {
+ data, marshalErr := json.Marshal(target)
+ if marshalErr != nil {
+ return marshalErr
}
- return err
- }
- defer release()
- var command updater.SingleUpdateCommand
- if decodeErr := data.Decode(&command); decodeErr != nil {
- return errors.Join(decodeErr, actor.ErrJobPermanentFailure)
- }
- var state updater.SingleUpdateState
- if getStateErr := s.singleUpdates.GetState(ctx, StateType, command.ActivityID, &state); getStateErr != nil {
- return getStateErr
- }
- if state.Status == "completed" || state.Status == "needs_attention" {
- return nil
- }
- if state.Command != command {
- return actor.ErrJobPermanentFailure
+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "container", ID: input.ContainerID, Status: scheduler.Running, RecoveryData: data, ActivityID: activityID})
}
- if authorizeSingleErr := s.authorizeSingleInternal(ctx, command); authorizeSingleErr != nil {
- return s.failSingleInternal(ctx, &state, authorizeSingleErr.Error())
+ if recordErr := record(); recordErr != nil {
+ return nil, recordErr
}
- ctx = s.trackActivity(ctx, command.ActivityID)
- canceled, err := s.awaitSingleActivityInternal(ctx, lifetime, &state)
- if err != nil || canceled {
- return err
- }
- completed, err := s.prepareSingleTargetInternal(ctx, &state)
- if err != nil || completed {
- return err
- }
- ctx = s.withFrozenTarget(ctx, state.Target, func() error {
- return s.singleUpdates.SetState(ctx, StateType, command.ActivityID, state, nil)
- })
+ ctx = s.deps.WithFrozenTarget(ctx, &target, record)
result, runErr := func() (out *updater.Result, err error) {
defer utils.RecoverToError(&err, "single container update")
- return s.runUpdate(ctx, state.Target.ContainerID, command.ActivityID)
+ return s.deps.RunUpdate(ctx, target.ContainerID, activityID)
}()
- return s.completeSingleInternal(ctx, &state, result, runErr)
-}
-
-func (s *Service) failSingleInternal(ctx context.Context, state *updater.SingleUpdateState, reason string) error {
- state.Status = "needs_attention"
- state.Failure = reason
- if err := s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil); err != nil {
- return err
- }
- s.finishUpdate(ctx, state.Command.ActivityID, nil, errors.New(reason))
- return actor.ErrJobPermanentFailure
-}
-
-func (s *Service) authorizeSingleInternal(ctx context.Context, command updater.SingleUpdateCommand) error {
- if command.UserID == "" {
- return nil
- }
- if s.config != nil && s.config.AgentMode && command.UserID == "agent" && command.KeyID == "" {
- return nil
- }
- if s.roles == nil {
- return errors.New("requesting user permissions unavailable")
- }
- permissions, err := s.roles.ResolveExecutionPermissions(ctx, command.UserID, command.KeyID)
- if err != nil || !permissions.Allows(authz.PermImageUpdatesCheck, "0") {
- return errors.New("requesting user no longer has permission to update containers")
- }
- return nil
-}
-
-func (s *Service) awaitSingleActivityInternal(ctx, lifetime context.Context, state *updater.SingleUpdateState) (bool, error) {
- if s.activity == nil {
- return false, nil
- }
- detail, err := s.activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)
- if err != nil {
- return false, err
- }
- if detail.Activity.Status == activitytypes.StatusCancelled {
- state.Status = "needs_attention"
- state.Failure = "Container update cancelled"
- return true, s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil)
- }
- err = s.activity.AwaitActivitySlotBounded(ctx, state.Command.ActivityID, "0")
- if err == nil {
- return false, nil
- }
- if ctx.Err() == nil {
- return false, s.failSingleInternal(ctx, state, err.Error())
- }
- if lifetime != nil && lifetime.Err() != nil {
- return false, ctx.Err()
- }
- state.Status = "needs_attention"
- state.Failure = err.Error()
- writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
- defer cancel()
- return true, s.singleUpdates.SetState(writeCtx, StateType, state.Command.ActivityID, *state, nil)
-}
-
-func (s *Service) prepareSingleTargetInternal(ctx context.Context, state *updater.SingleUpdateState) (bool, error) {
- if state.Status != "running" {
- target, err := s.freezeContainer(ctx, state.Command.ContainerID)
- if err != nil {
- return false, s.failSingleInternal(ctx, state, err.Error())
- }
- state.Target = target
- state.Status = "running"
- return false, s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil)
- }
- if state.Target == nil {
- return false, s.failSingleInternal(ctx, state, "Interrupted container update has no saved target evidence")
- }
- confirmed, unchanged, err := s.confirmTarget(ctx, *state.Target)
- if err != nil {
- return false, err
- }
- if confirmed {
- result := &updater.Result{
- Updated: 1,
- Items: []updater.ResourceResult{
- {
- ResourceID: state.Target.ContainerID,
- ResourceName: state.Target.ContainerName,
- ResourceType: "container",
- Status: updater.StatusUpdated,
- UpdateApplied: true,
- },
- },
- }
- return true, s.completeSingleInternal(ctx, state, result, nil)
- }
- if !unchanged {
- return false, s.failSingleInternal(ctx, state, "Interrupted container update effect cannot be confirmed")
- }
- return false, nil
-}
-
-func (s *Service) completeSingleInternal(ctx context.Context, state *updater.SingleUpdateState, result *updater.Result, runErr error) error {
if ctx.Err() != nil {
- return ctx.Err()
+ return nil, ctx.Err()
}
- if runErr != nil {
- return s.failSingleInternal(ctx, state, runErr.Error())
- }
- if state.Target.DesiredImageRef != "" && result != nil && result.Updated > 0 {
- confirmed, _, err := s.confirmTarget(ctx, *state.Target)
- if err != nil {
- return err
+ if runErr == nil && target.DesiredImageRef != "" && result != nil && result.Updated > 0 {
+ confirmed, _, confirmErr := s.deps.ConfirmTarget(ctx, target)
+ if confirmErr != nil {
+ return nil, confirmErr
}
if !confirmed {
- return s.failSingleInternal(ctx, state, "Updater result does not confirm the frozen desired image")
+ return failed("Updater result does not confirm the frozen desired image"), nil
}
}
- state.Status = "completed"
- state.Result = result
- if err := s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil); err != nil {
- return err
- }
- s.finishUpdate(ctx, state.Command.ActivityID, result, nil)
- return nil
-}
-
-func (s *Service) ActiveUpdateActivityIDs(ctx context.Context) ([]string, error) {
- ids := []string{}
- for cursor := ""; ; {
- page, err := s.singleUpdates.ListStates(ctx, StateType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
- if err != nil {
- return nil, err
- }
- for _, entry := range page.States {
- // States are keyed by activity ID, so an undecodable entry is still protected.
- if _, decodeErr := decodeSingleStateInternal(entry); decodeErr != nil {
- s.logger().WarnContext(ctx, "decode container update state", "activityId", entry.ActorID, "error", decodeErr)
- }
- ids = append(ids, entry.ActorID)
- }
- cursor = page.AfterID()
- if cursor == "" {
- return ids, nil
- }
- }
-}
-
-func singleUpdateCommandInternal(ctx context.Context, containerID, activityID string) updater.SingleUpdateCommand {
- command := updater.SingleUpdateCommand{ContainerID: containerID, ActivityID: activityID}
- if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {
- command.UserID = user.ID
- }
- command.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)
- return command
+ return s.deps.RecordUpdate(ctx, activityID, result, runErr), nil
}
diff --git a/backend/internal/updater/children/execution/service_test.go b/backend/internal/updater/children/execution/service_test.go
deleted file mode 100644
index 0f79a4034..000000000
--- a/backend/internal/updater/children/execution/service_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package execution
-
-import (
- "context"
- "errors"
- "log/slog"
- "testing"
- "time"
-
- "github.com/getarcaneapp/arcane/types/v2/updater"
- "github.com/stretchr/testify/require"
-
- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
-)
-
-// testDependenciesInternal wires delivery to stand-ins: freezing fails as it
-// does without Docker.
-func testDependenciesInternal() Dependencies {
- return Dependencies{
- Logger: slog.Default,
- AcquireUpdate: func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },
- UpdateBusy: errors.New("busy"),
- TrackActivity: func(ctx context.Context, _ string) context.Context { return ctx },
- RunUpdate: func(context.Context, string, string) (*updater.Result, error) {
- return nil, errors.New("unexpected update run")
- },
- FinishUpdate: func(context.Context, string, *updater.Result, error) {},
- FreezeContainer: func(context.Context, string) (*updater.FrozenUpdateTarget, error) {
- return nil, errors.New("docker service unavailable")
- },
- ConfirmTarget: func(context.Context, updater.FrozenUpdateTarget) (bool, bool, error) { return false, false, nil },
- WithFrozenTarget: func(ctx context.Context, _ *updater.FrozenUpdateTarget, _ func() error) context.Context {
- return ctx
- },
- }
-}
-
-func TestService_RepairsAcceptedIntentBeforeDispatch(t *testing.T) {
- svc := NewService(testDependenciesInternal())
- runtime := francistest.New(t)
- require.NoError(t, svc.RegisterActors(runtime))
- francistest.Start(t, runtime)
- command := updater.SingleUpdateCommand{ContainerID: "missing", ActivityID: "accepted"}
- require.NoError(
- t,
- runtime.Service().SetState(
- t.Context(),
- StateType,
- command.ActivityID,
- updater.SingleUpdateState{
- Command: command,
- Status: "queued",
- },
- nil,
- ),
- )
- require.NoError(t, svc.Start(t.Context()))
- t.Cleanup(func() {
- ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), time.Second)
- defer cancel()
- require.NoError(t, svc.Stop(ctx))
- })
- require.Eventually(t, func() bool {
- var state updater.SingleUpdateState
- return runtime.Service().GetState(t.Context(), StateType, command.ActivityID, &state) == nil && state.Status == "needs_attention"
- }, 3*time.Second, 10*time.Millisecond)
-}
diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.go
index a1503e508..db440c3ee 100644
--- a/backend/internal/updater/children/recovery/service.go
+++ b/backend/internal/updater/children/recovery/service.go
@@ -1,6 +1,7 @@
package recovery
import (
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -96,29 +97,27 @@ type frozenSingleInternal struct {
Persist func() error
}
-// FreezePending persists identities and desired images before any pull.
-func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {
+// errNoFrozenPlanInternal reports a run without a persisted frozen plan.
+var errNoFrozenPlanInternal = errors.New("interrupted update has no frozen target plan")
+
+// FreezePending persists identities and desired images before any pull. A run
+// that already holds a plan keeps it. Image refs and container IDs in excluded are left
+// out, and a target that cannot be frozen fails alone instead of aborting the batch.
+func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excluded map[string]bool) (context.Context, error) {
if _, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal); ok {
return ctx, nil
}
- run, ok := jobcontext.Run(ctx)
- if !ok {
- return ctx, nil
- }
- for _, target := range run.Outcome.Targets {
- if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
- var plan frozenUpdatePlanInternal
- if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
- return ctx, err
- }
- return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil
- }
+ if plan, found, err := frozenPlanInternal(run); err != nil || found {
+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), err
}
records, err := s.pendingUpdates(ctx)
if err != nil {
return ctx, err
}
- plan, err := s.buildFrozenPlanInternal(ctx, records)
+ records = slices.DeleteFunc(records, func(record updater.ImageUpdateRecord) bool {
+ return excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]
+ })
+ plan, failures, err := s.buildFrozenPlanInternal(ctx, records, excluded)
if err != nil {
return ctx, err
}
@@ -126,18 +125,16 @@ func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {
if err != nil {
return ctx, err
}
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ID: "auto-update",
- ResourceType: "update-batch",
- Status: scheduler.Running,
- RecoveryData: raw,
- ActivityID: s.activityID(
- ctx,
- ),
- },
- ); progressErr != nil {
+ activityID := s.activityID(ctx)
+ // Failures are recorded before the plan, since a delivery that finds the plan never plans again.
+ for containerID, freezeErr := range failures {
+ failure := scheduler.TargetOutcome{ID: containerID, ResourceType: "container", Status: scheduler.Failed, Message: "Update could not be planned: " + freezeErr.Error(), ActivityID: activityID}
+ if failedProgressErr := jobcontext.Progress(ctx, failure); failedProgressErr != nil {
+ return ctx, failedProgressErr
+ }
+ }
+ batch := scheduler.TargetOutcome{ID: "auto-update", ResourceType: "update-batch", Status: scheduler.Running, RecoveryData: raw, ActivityID: activityID}
+ if progressErr := jobcontext.Progress(ctx, batch); progressErr != nil {
return ctx, progressErr
}
for _, target := range plan.Targets {
@@ -148,25 +145,26 @@ func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {
return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil
}
-func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater.ImageUpdateRecord) (frozenUpdatePlanInternal, error) {
+func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater.ImageUpdateRecord, excluded map[string]bool) (frozenUpdatePlanInternal, map[string]error, error) {
plan := frozenUpdatePlanInternal{Records: make([]updater.ImageUpdateRecord, 0, len(records))}
+ failures := map[string]error{}
if len(records) == 0 {
- return plan, nil
+ return plan, failures, nil
}
dockerClient, err := s.dockerClient(ctx)
if err != nil {
- return plan, err
+ return plan, nil, err
}
listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
if err != nil {
- return plan, err
+ return plan, nil, err
}
for _, record := range records {
if !record.HasUpdate {
continue
}
for _, candidate := range listed.Items {
- if record.ContainerID != "" && record.ContainerID != candidate.ID {
+ if excluded[candidate.ID] || (record.ContainerID != "" && record.ContainerID != candidate.ID) {
continue
}
if record.ContainerID == "" && refs.NormalizeImageUpdateRef(candidate.Image) != refs.NormalizeImageUpdateRef(record.ImageRef()) {
@@ -174,13 +172,14 @@ func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater
}
target, selected, freezeRecordTargetErr := s.freezeRecordTargetInternal(ctx, record, candidate.ID)
if freezeRecordTargetErr != nil {
- return plan, freezeRecordTargetErr
+ failures[candidate.ID] = freezeRecordTargetErr
+ continue
}
plan.Records = append(plan.Records, selected)
plan.Targets = append(plan.Targets, *target)
}
}
- return plan, nil
+ return plan, failures, nil
}
func (s *Service) freezeRecordTargetInternal(ctx context.Context, record updater.ImageUpdateRecord, containerID string) (*arcaneupdater.FrozenUpdateTarget, updater.ImageUpdateRecord, error) {
@@ -364,18 +363,40 @@ func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (sche
return scheduler.Outcome{Status: scheduler.Waiting}, err
}
defer release()
- var plan frozenUpdatePlanInternal
- found := false
- for _, target := range run.Outcome.Targets {
- if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
- if unmarshalErr := json.Unmarshal(target.RecoveryData, &plan); unmarshalErr != nil {
- return scheduler.Outcome{Status: scheduler.Failed}, unmarshalErr
- }
- found = true
+ ctx, remaining, unresolved, err := s.ResumePlan(ctx, run)
+ if errors.Is(err, errNoFrozenPlanInternal) {
+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil
+ }
+ if err != nil {
+ return scheduler.Outcome{Status: scheduler.Waiting}, err
+ }
+ if remaining > 0 {
+ result, applyPendingErr := s.applyPending(ctx, arcaneupdater.Options{})
+ if applyPendingErr != nil || result == nil || result.Failed > 0 {
+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Frozen update recovery could not confirm completion"}, applyPendingErr
}
}
+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Frozen update targets confirmed"}
+ // Containers that could not be planned were never updated, so recovery reports them as finalize does.
+ if outcome.Targets = s.Unplanned(run); len(outcome.Targets) > 0 {
+ outcome.Status, outcome.Message = scheduler.Partial, "Some updates failed"
+ }
+ if unresolved {
+ outcome.Status, outcome.Message = scheduler.Failed, "Some update effects could not be confirmed"
+ }
+ return outcome, nil
+}
+
+// ResumePlan confirms each unsettled frozen target before anything repeats and
+// scopes ctx to the targets still to apply. Targets that changed without
+// reaching the desired image are marked for review and reported as unresolved.
+func (s *Service) ResumePlan(ctx context.Context, run scheduler.Run) (context.Context, int, bool, error) {
+ plan, found, err := frozenPlanInternal(run)
+ if err != nil {
+ return ctx, 0, false, err
+ }
if !found {
- return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil
+ return ctx, 0, false, errNoFrozenPlanInternal
}
remaining := frozenUpdatePlanInternal{}
unresolved := false
@@ -383,44 +404,85 @@ func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (sche
if frozenTargetSettledInternal(run, target.ContainerID) {
continue
}
- confirmed, unchanged, confirmFrozenTargetErr := s.ConfirmTarget(ctx, target)
- if confirmFrozenTargetErr != nil {
- return scheduler.Outcome{Status: scheduler.Waiting}, confirmFrozenTargetErr
- }
- if confirmed {
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ID: target.ContainerID,
- ResourceType: "container",
- Status: scheduler.Succeeded,
- Message: "Frozen desired image confirmed after restart",
- },
- ); progressErr != nil {
- return scheduler.Outcome{}, progressErr
- }
- continue
+ confirmed, unchanged, confirmErr := s.ConfirmTarget(ctx, target)
+ if confirmErr != nil {
+ return ctx, 0, false, confirmErr
}
- if !unchanged {
+ progress := scheduler.TargetOutcome{ID: target.ContainerID, ResourceType: "container"}
+ switch {
+ case confirmed:
+ progress.Status, progress.Message = scheduler.Succeeded, "Frozen desired image confirmed"
+ case !unchanged:
unresolved = true
+ progress.Status, progress.Message = scheduler.NeedsAttention, "Update effect could not be confirmed"
+ default:
+ remaining.Targets = append(remaining.Targets, target)
+ remaining.Records = appendFrozenRecordInternal(remaining.Records, plan.Records, target.ContainerID)
continue
}
- remaining.Targets = append(remaining.Targets, target)
- remaining.Records = appendFrozenRecordInternal(remaining.Records, plan.Records, target.ContainerID)
+ if progressErr := jobcontext.Progress(ctx, progress); progressErr != nil {
+ return ctx, 0, false, progressErr
+ }
}
- if len(remaining.Records) > 0 {
- result, applyPendingErr := s.applyPending(context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), arcaneupdater.Options{})
- if applyPendingErr != nil || result == nil || result.Failed > 0 {
- return scheduler.Outcome{Status: scheduler.Failed, Message: "Frozen update recovery could not confirm completion"}, applyPendingErr
+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), len(remaining.Records), unresolved, nil
+}
+
+// Planned reports whether the run already holds a frozen plan, as a retry does.
+func (s *Service) Planned(run scheduler.Run) bool {
+ _, found, err := frozenPlanInternal(run)
+ return found && err == nil
+}
+
+// Started reports whether applying the frozen plan began: a planned container moved past queued.
+func (s *Service) Started(run scheduler.Run) bool {
+ plan, found, err := frozenPlanInternal(run)
+ if err != nil || !found {
+ return false
+ }
+ return slices.ContainsFunc(plan.Targets, func(planned arcaneupdater.FrozenUpdateTarget) bool {
+ return slices.ContainsFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == planned.ContainerID && target.Status != scheduler.Queued
+ })
+ })
+}
+
+// Unsettled counts the frozen targets a retry would still resume.
+func (s *Service) Unsettled(run scheduler.Run) (int, error) {
+ plan, found, err := frozenPlanInternal(run)
+ if err != nil || !found {
+ return 0, cmp.Or(err, errNoFrozenPlanInternal)
+ }
+ unsettled := 0
+ for _, target := range plan.Targets {
+ if !frozenTargetSettledInternal(run, target.ContainerID) {
+ unsettled++
}
}
- status := scheduler.Succeeded
- message := "Frozen update targets confirmed"
- if unresolved {
- status = scheduler.Failed
- message = "Some update effects could not be confirmed"
+ return unsettled, nil
+}
+
+// Unplanned lists the images and containers that never reached the frozen plan: failed checks and failed freezes.
+func (s *Service) Unplanned(run scheduler.Run) []scheduler.TargetOutcome {
+ plan, found, err := frozenPlanInternal(run)
+ if err != nil || !found {
+ return nil
+ }
+ return slices.DeleteFunc(slices.Clone(run.Outcome.Targets), func(target scheduler.TargetOutcome) bool {
+ return (target.ResourceType != "container" && target.ResourceType != "image") || target.Status != scheduler.Failed ||
+ slices.ContainsFunc(plan.Targets, func(planned arcaneupdater.FrozenUpdateTarget) bool { return planned.ContainerID == target.ID })
+ })
+}
+
+// frozenPlanInternal decodes the plan persisted on the run's batch target.
+func frozenPlanInternal(run scheduler.Run) (frozenUpdatePlanInternal, bool, error) {
+ var plan frozenUpdatePlanInternal
+ for _, target := range run.Outcome.Targets {
+ if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
+ err := json.Unmarshal(target.RecoveryData, &plan)
+ return plan, err == nil, err
+ }
}
- return scheduler.Outcome{Status: status, Message: message}, nil
+ return plan, false, nil
}
func appendFrozenRecordInternal(remaining, records []updater.ImageUpdateRecord, id string) []updater.ImageUpdateRecord {
diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.go
index 96061cf32..f3d1a567e 100644
--- a/backend/internal/updater/children/recovery/service_test.go
+++ b/backend/internal/updater/children/recovery/service_test.go
@@ -100,7 +100,7 @@ func TestFrozenTargetsPreserveAllSelectedContainers(t *testing.T) {
}
return nil
})
- frozenCtx, err := svc.FreezePending(ctx)
+ frozenCtx, err := svc.FreezePending(ctx, scheduler.Run{ID: "run"}, nil)
require.NoError(t, err)
plan := frozenCtx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)
require.Len(t, plan.Records, 2)
@@ -170,3 +170,109 @@ func TestFrozenTargetConfirmsReplacementAndRejectsUnknownEffect(t *testing.T) {
})
}
}
+
+func TestFreezePendingExcludesFailedChecksAndIsolatesFreezeFailures(t *testing.T) {
+ desired := "sha256:desired"
+ pending := []updater.ImageUpdateRecord{
+ {ID: "app", Repository: "registry.example.com/app", Tag: "latest", HasUpdate: true, LatestDigest: &desired},
+ {ID: "unresolvable", Repository: "registry.example.com/unresolvable", Tag: "latest", HasUpdate: true},
+ {ID: "unchecked", Repository: "registry.example.com/unchecked", Tag: "latest", HasUpdate: true, LatestDigest: &desired},
+ }
+ running := []container.Summary{
+ {ID: "app", Names: []string{"/app"}, Image: "registry.example.com/app:latest"},
+ {ID: "unresolvable", Names: []string{"/unresolvable"}, Image: "registry.example.com/unresolvable:latest"},
+ {ID: "unchecked", Names: []string{"/unchecked"}, Image: "registry.example.com/unchecked:latest"},
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if strings.HasSuffix(r.URL.Path, "/containers/json") {
+ assert.NoError(t, json.MarshalWrite(w, running))
+ return
+ }
+ for _, summary := range running {
+ if strings.HasSuffix(r.URL.Path, "/containers/"+summary.ID+"/json") {
+ assert.NoError(t, json.MarshalWrite(w, container.InspectResponse{
+ ID: summary.ID, Name: summary.Names[0], Image: "sha256:old",
+ Config: &container.Config{Image: summary.Image}, State: &container.State{StartedAt: "baseline"},
+ }))
+ return
+ }
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+ svc := newTestServiceInternal(t, server, func(context.Context) ([]updater.ImageUpdateRecord, error) { return pending, nil })
+ targets := map[string]scheduler.TargetOutcome{}
+ ctx := jobcontext.WithExecution(t.Context(), scheduler.Run{ID: "run"}, func(target scheduler.TargetOutcome) error {
+ targets[target.ID] = target
+ return nil
+ })
+
+ frozenCtx, err := svc.FreezePending(ctx, scheduler.Run{ID: "run"}, map[string]bool{"registry.example.com/unchecked:latest": true})
+ require.NoError(t, err)
+ plan := frozenCtx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)
+ require.Len(t, plan.Targets, 1)
+ require.Equal(t, "app", plan.Targets[0].ContainerID)
+ require.Equal(t, scheduler.Queued, targets["app"].Status)
+ require.Equal(t, scheduler.Failed, targets["unresolvable"].Status, "a target that cannot be frozen fails alone")
+ require.Contains(t, targets["unresolvable"].Message, "Update could not be planned")
+ require.NotContains(t, targets, "unchecked", "images whose check failed stay out of the plan")
+}
+
+func TestResumePlanConfirmsRedeliveredChangeWithoutReapplying(t *testing.T) {
+ target := arcaneupdater.FrozenUpdateTarget{
+ ContainerID: "old", ContainerName: "app", BaselineImageID: "sha256:baseline", BaselineStartedAt: "baseline",
+ DesiredImageRef: "registry.example.com/app:latest", DesiredDigest: "sha256:desired",
+ }
+ raw, err := json.Marshal(frozenUpdatePlanInternal{
+ Records: []updater.ImageUpdateRecord{{ID: "image", ContainerID: "old", Repository: "registry.example.com/app", Tag: "latest", HasUpdate: true}},
+ Targets: []arcaneupdater.FrozenUpdateTarget{target},
+ })
+ require.NoError(t, err)
+ run := scheduler.Run{ID: "run", Outcome: scheduler.Outcome{Targets: []scheduler.TargetOutcome{
+ {ID: "auto-update", ResourceType: "update-batch", Status: scheduler.Running, RecoveryData: raw},
+ {ID: "old", ResourceType: "container", Status: scheduler.Running},
+ }}}
+ for _, test := range []struct {
+ name, id, image string
+ remaining int
+ status scheduler.RunStatus
+ }{
+ {"applied before the redelivery", "new", "sha256:desired", 0, scheduler.Succeeded},
+ {"not yet applied", "old", "sha256:baseline", 1, ""},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ assert.NoError(t, json.MarshalWrite(w, []container.Summary{{ID: test.id, Names: []string{"/app"}}}))
+ case strings.Contains(r.URL.Path, "/images/"):
+ assert.NoError(t, json.MarshalWrite(w, image.InspectResponse{ID: "sha256:desired"}))
+ case strings.Contains(r.URL.Path, "/containers/"):
+ assert.NoError(t, json.MarshalWrite(w, container.InspectResponse{
+ ID: test.id, Name: "/app", Image: test.image, Config: &container.Config{}, State: &container.State{StartedAt: "baseline"},
+ }))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ svc := newTestServiceInternal(t, server, nil)
+ var status scheduler.RunStatus
+ ctx := jobcontext.WithExecution(t.Context(), run, func(progress scheduler.TargetOutcome) error {
+ status = progress.Status
+ return nil
+ })
+
+ resumed, remaining, unresolved, resumeErr := svc.ResumePlan(ctx, run)
+ require.NoError(t, resumeErr)
+ require.False(t, unresolved)
+ require.Equal(t, test.remaining, remaining)
+ require.Equal(t, test.status, status)
+ records, ok := svc.FrozenRecords(resumed)
+ require.True(t, ok)
+ require.Len(t, records, test.remaining)
+ })
+ }
+}
diff --git a/backend/internal/updater/handler.go b/backend/internal/updater/handler.go
index 0d8f23c75..2c4e50883 100644
--- a/backend/internal/updater/handler.go
+++ b/backend/internal/updater/handler.go
@@ -102,7 +102,7 @@ func (h *UpdaterHandler) GetUpdaterHistory(ctx context.Context, input *GetUpdate
}, nil
}
-func (h *UpdaterHandler) updateContainerInternal(ctx context.Context, input *UpdateContainerInput) (*updateContainerOutput, error) {
+func (h *UpdaterHandler) updateContainer(ctx context.Context, input *UpdateContainerInput) (*updateContainerOutput, error) {
runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
if input.Async {
acceptedActivity, err := h.updaterService.AcceptSingleContainerUpdate(runtimeCtx, input.ContainerID)
diff --git a/backend/internal/updater/module.go b/backend/internal/updater/module.go
index 93c2f8ebd..74eeadf8d 100644
--- a/backend/internal/updater/module.go
+++ b/backend/internal/updater/module.go
@@ -38,11 +38,7 @@ func (m *Module) Service() *UpdaterService {
// RegisterRoutes mounts the updater endpoints. A nil module still registers, so
// OpenAPI spec generation can discover the routes without a service graph.
func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {
- if m == nil {
- RegisterUpdater(api, nil, appCtx)
- return
- }
- RegisterUpdater(api, m.service, appCtx)
+ RegisterUpdater(api, m.Service(), appCtx)
}
// RegisterUpdater registers updater management routes using Huma.
@@ -101,5 +97,5 @@ func RegisterUpdater(api huma.API, updaterService *UpdaterService, appCtx handle
"200": {Description: "Container update completed", Content: map[string]*huma.MediaType{"application/json": {Schema: completedSchema}}},
"202": {Description: "Container update accepted", Content: map[string]*huma.MediaType{"application/json": {Schema: acceptedSchema}}},
},
- }, authz.PermImageUpdatesCheck, h.updateContainerInternal)
+ }, authz.PermImageUpdatesCheck, h.updateContainer)
}
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index c9fb41c06..32b527981 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -17,16 +17,18 @@ import (
"github.com/containerd/errdefs"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
+ imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"
projecttypes "github.com/getarcaneapp/arcane/types/v2/project"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/samber/mo"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/cgroup"
"go.getarcane.app/updater"
"go.getarcane.app/updater/labels"
@@ -53,10 +55,10 @@ import (
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
@@ -64,22 +66,18 @@ import (
// UpdaterService is Arcane's handler-facing service for the standalone updater engine.
type UpdaterService struct {
- admission *runs.Admission
- config *config.Config
- coordinator *runs.Coordinator
- roles *role.RoleService
- deps updaterDependenciesInternal
- engine *updater.Service
- execution *execution.Service
- recovery *recovery.Service
- // updateMu serializes per-container updates. docker compose's recreate
- // pipeline is not concurrency-safe for sibling containers sharing a
- // namespace. ponytail: global lock ceiling — all updates serialize; fine
- // for a UI, upgrade to per-project if batch throughput ever matters.
+ admission *runs.Admission
+ deps updaterDependencies
+ engine *updater.Service
+ execution *execution.Service
+ recovery *recovery.Service
+ autoUpdate *flow.Workflow
+ // updateMu serializes container updates: compose's recreate pipeline is not safe for
+ // sibling containers sharing a namespace. Upgrade to per-project if throughput matters.
updateMu sync.Mutex
}
-type updaterDependenciesInternal struct {
+type updaterDependencies struct {
DB *database.DB
Docker *dockerInternal.DockerClientService
Settings *settings.SettingsService
@@ -89,18 +87,66 @@ type updaterDependenciesInternal struct {
RegistryDigestResolver *registry.ContainerRegistryService
Events *event.EventService
Notifications *notification.NotificationService
- SelfUpgrade selfUpgradeServiceInternal
+ SelfUpgrade selfUpgradeService
Activity *activity.ActivityService
SystemUser usertypes.Actor
- Logger *slog.Logger
}
-type selfUpgradeServiceInternal interface {
+type selfUpgradeService interface {
// TriggerUpgradeViaCLI returns the spawned upgrader container's ID, which this
// service does not need — only update-all's manager step uses it.
TriggerUpgradeViaCLI(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error)
}
+type (
+ // containerUpdateBatch accumulates per-container update notifications
+ // so a single update batch produces one batched notification.
+ containerUpdateBatch struct {
+ sync.Mutex
+
+ entries []notifications.ContainerUpdateBatchEntry
+ }
+
+ containerUpdateBatchContextKey struct{}
+ activityIDContextKey struct{}
+ updateAdmissionKey struct{}
+ updateProgressKey struct{}
+
+ updateProgress struct {
+ mu sync.Mutex
+ err error
+ selfTriggered bool
+ selfID string
+ }
+
+ // autoUpdateApplied is the apply step's output, read back by finalize.
+ autoUpdateApplied struct {
+ Result arcaneupdater.Result `json:"result"`
+ Error string `json:"error,omitempty"`
+ RecordingError string `json:"recordingError,omitempty"`
+ SelfTriggered bool `json:"selfTriggered,omitempty"`
+ SelfID string `json:"selfId,omitempty"`
+ Unresolved bool `json:"unresolved,omitempty"`
+ Completed bool `json:"completed"`
+ // Busy means another update held the updater, so this run applied nothing; Started means an earlier
+ // delivery had begun applying its plan.
+ Busy bool `json:"busy,omitempty"`
+ Started bool `json:"started,omitempty"`
+ }
+)
+
+var (
+ errUpdateBusy = errors.New("another container update is running")
+
+ containerEventTypes = map[string]event.EventType{
+ "container_stop": event.EventTypeContainerStop,
+ "container_delete": event.EventTypeContainerDelete,
+ "container_create": event.EventTypeContainerCreate,
+ "container_start": event.EventTypeContainerStart,
+ "container_update": event.EventTypeContainerUpdate,
+ }
+)
+
// NewUpdaterService constructs the Arcane updater facade.
func NewUpdaterService(
db *database.DB,
@@ -112,16 +158,16 @@ func NewUpdaterService(
events *event.EventService,
imageSvc *image.ImageService,
localNotifications *notification.NotificationService,
- upgrade selfUpgradeServiceInternal,
+ upgrade selfUpgradeService,
activityService *activity.ActivityService,
cfg *config.Config,
- coordinator *runs.Coordinator,
+ _ *runs.Coordinator,
admission *runs.Admission,
roles *role.RoleService,
) (*UpdaterService, error) {
service := &UpdaterService{
- config: cfg, coordinator: coordinator, admission: admission, roles: roles,
- deps: updaterDependenciesInternal{
+ admission: admission,
+ deps: updaterDependencies{
DB: db,
Docker: localDocker,
Settings: localSettings,
@@ -136,30 +182,49 @@ func NewUpdaterService(
SystemUser: usertypes.SystemUser,
},
}
+ // A nil registry service must reach the engine as a nil interface, not a typed nil.
+ var digestResolver updater.RegistryDigestResolver
+ if registries != nil {
+ digestResolver = registries
+ }
service.recovery = recovery.NewService(
service.DockerClient,
- service.registryDigestResolverInternal,
+ func() updater.RegistryDigestResolver { return digestResolver },
service.PendingImageUpdates,
- activityIDFromContextInternal,
- service.acquireUpdateInternal,
+ activityIDFromContext,
+ service.acquireUpdate,
service.ApplyPending,
)
service.execution = execution.NewService(execution.Dependencies{
- Coordinator: coordinator,
Config: cfg,
Roles: roles,
- Activity: activityService,
- Logger: service.loggerInternal,
- AcquireUpdate: service.acquireUpdateInternal,
- UpdateBusy: errUpdateBusyInternal,
- TrackActivity: service.trackActivityInternal,
- RunUpdate: service.runSingleContainerUpdateInternal,
- FinishUpdate: service.finishSingleContainerUpdateInternal,
+ AcquireUpdate: service.acquireUpdate,
+ UpdateBusy: errUpdateBusy,
+ RunUpdate: service.runSingleContainerUpdate,
+ RecordUpdate: service.recordSingleContainerUpdate,
FreezeContainer: service.recovery.FreezeContainer,
ConfirmTarget: service.recovery.ConfirmTarget,
WithFrozenTarget: service.recovery.WithSingleTarget,
})
- engine, err := updater.New(service.configInternal())
+ // The self container ID routes Arcane through the CLI self-updater even without its labels.
+ selfContainerID, selfErr := cgroup.CurrentContainerID()
+ engine, err := updater.New(updater.Config{
+ DockerClientProvider: service,
+ ImagePuller: service,
+ PendingStore: service,
+ RunRecorder: service,
+ Settings: service,
+ RegistryDigestResolver: digestResolver,
+ RegistryTagLister: service,
+ ProjectUpdater: service,
+ SelfUpdater: service,
+ Notifier: service,
+ EventRecorder: service,
+ UsedImageCollector: updater.UsedImageCollectorFunc(service.CollectUsedImages),
+ LabelPolicy: updater.DefaultLabelPolicy(),
+ SelfContainerID: kit.Ternary(selfErr != nil, "", selfContainerID),
+ Logger: slog.Default(),
+ })
if err != nil {
return nil, fmt.Errorf("configure updater engine: %w", err)
}
@@ -168,109 +233,88 @@ func NewUpdaterService(
return service, nil
}
-func (s *UpdaterService) configInternal() updater.Config {
- return updater.Config{
- DockerClientProvider: s,
- ImagePuller: s,
- PendingStore: s,
- RunRecorder: s,
- Settings: s,
- RegistryDigestResolver: s.registryDigestResolverInternal(),
- RegistryTagLister: s,
- ProjectUpdater: s,
- SelfUpdater: s,
- Notifier: s,
- EventRecorder: s,
- UsedImageCollector: updater.UsedImageCollectorFunc(s.CollectUsedImages),
- LabelPolicy: updater.DefaultLabelPolicy(),
- SelfContainerID: selfContainerIDInternal(),
- Logger: s.loggerInternal(),
- }
-}
-
-// selfContainerIDInternal returns the ID of the container Arcane runs in, so
-// the updater engine routes it through the CLI self-updater even when the
-// container is missing the Arcane labels. Empty when not running in Docker.
-func selfContainerIDInternal() string {
- id, err := cgroup.CurrentContainerID()
- return kit.Ternary(err != nil, "", id)
-}
-
-func (s *UpdaterService) engineInternal() *updater.Service {
- return s.engine
-}
-
-func (s *UpdaterService) loggerInternal() *slog.Logger {
- if s.deps.Logger != nil {
- return s.deps.Logger
- }
- return slog.Default()
-}
-
-func (s *UpdaterService) registryDigestResolverInternal() updater.RegistryDigestResolver {
- if s == nil || s.deps.RegistryDigestResolver == nil {
- return nil
- }
- return s.deps.RegistryDigestResolver
-}
-
-// ApplyPending executes pending image updates. When the options carry
-// resource IDs the run is scoped: the engine's ApplyPending has no resource
-// filtering (it would apply every pending update), so scoped requests resolve
-// to concrete containers and go through the engine's single-container path
-// instead — same activity, events, and cleanup either way.
+// ApplyPending executes pending image updates. The engine's ApplyPending applies every
+// pending update, so a scoped request runs its containers through the single-container path.
func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {
var release func()
- ctx, release, err = s.acquireUpdateInternal(ctx)
+ ctx, release, err = s.acquireUpdate(ctx)
if err != nil {
return nil, err
}
defer release()
start := time.Now()
batchCompleted := false
- activityID := s.startAutoUpdateActivityInternal(ctx, options.DryRun)
- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
- ctx = s.trackActivityInternal(ctx, activityID)
- ctx = contextWithActivityIDInternal(ctx, activityID)
- progress := &updateProgressInternal{}
- ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)
- notifyBatch := &containerUpdateBatchInternal{}
- ctx = context.WithValue(ctx, containerUpdateBatchContextKeyInternal{}, notifyBatch)
+ activityID := ""
+ if s.deps.Activity != nil {
+ started, startErr := s.deps.Activity.StartActivity(ctx, activitylib.StartRequest{
+ EnvironmentID: "0",
+ Type: activitytypes.TypeAutoUpdate,
+ Queue: true,
+ ResourceType: new("system"),
+ ResourceName: new("Auto update"),
+ Step: "Planning updates",
+ LatestMessage: "Auto-update run started",
+ Metadata: database.JSON{"dryRun": options.DryRun},
+ })
+ if startErr != nil {
+ slog.DebugContext(ctx, "failed to start auto-update activity", "error", startErr)
+ } else {
+ activityID = started.ID
+ ctx = contextWithActivityID(s.deps.Activity.Track(ctx, activityID), activityID)
+ }
+ }
+ out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}
+ progress := &updateProgress{}
+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)
defer func() {
- s.flushBatchedContainerUpdatesInternal(ctx, notifyBatch)
- if out == nil {
- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}
- }
- if out.Duration == "" {
- out.Duration = time.Since(start).String()
+ out.Duration = cmp.Or(out.Duration, time.Since(start).String())
+ out.ActivityID = mo.EmptyableToOption(activityID).ToPointer()
+ err = progress.completeBatch(ctx, options, out, batchCompleted, err)
+ failure := ""
+ if err != nil {
+ failure = err.Error()
+ } else if out.Failed > 0 {
+ failure = fmt.Sprintf("%d update action(s) failed", out.Failed)
}
- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- err = progress.completeBatchInternal(ctx, options, out, batchCompleted, err)
- s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)
+ s.completeAutoUpdateActivity(ctx, activityID, "Auto-update run completed", failure)
}()
- ctx, err = s.recovery.FreezePending(ctx)
- if err != nil {
- return out, err
- }
-
- if activityID != "" && s.deps.Activity != nil {
- // Bounded slot wait: an unbounded wait behind other long-running runs
- // would strand the queued activity row (the completion defer above
- // flips it to failed on timeout instead).
+ if activityID != "" {
+ // Bounded so a long queue fails the activity instead of stranding it queued.
if slotErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {
return out, slotErr
}
}
- // The engine's per-container docker operations carry no timeouts, so cap
- // the whole run; the Track ctx stays unbounded for the deferred completion
- // so user cancellation is still detected there.
+ if applyErr := s.applyBatch(ctx, options, out); applyErr != nil {
+ return out, applyErr
+ }
+ batchCompleted = true
+ return out, nil
+}
+
+// applyBatch runs one update batch into out, records its events, and sends its notifications
+// as one batch. The engine's per-container docker operations carry no timeouts, so the run is capped.
+func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {
+ batch := &containerUpdateBatch{}
+ ctx = context.WithValue(ctx, containerUpdateBatchContextKey{}, batch)
+ defer func() {
+ batch.Lock()
+ entries := batch.entries
+ batch.Unlock()
+ if len(entries) == 0 || s.deps.Notifications == nil {
+ return
+ }
+ if err := s.deps.Notifications.SendBatchContainerUpdateNotification(ctx, entries); err != nil {
+ slog.ErrorContext(ctx, "failed to send batched container update notification", "error", err, "count", len(entries))
+ }
+ }()
runCtx, cancelRun := context.WithTimeout(ctx, timeouts.DefaultAutoUpdateApply)
defer cancelRun()
+ activityID := activityIDFromContext(ctx)
- s.recordAutoUpdateEventInternal(ctx, event.EventSeverityInfo, database.JSON{
+ s.recordAutoUpdateEvent(ctx, event.EventSeverityInfo, database.JSON{
"phase": "start",
"dryRun": options.DryRun,
"forceUpdate": options.ForceUpdate,
@@ -278,36 +322,33 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater
"scopedCount": len(options.ResourceIds),
"time": time.Now().UTC().Format(time.RFC3339),
})
- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Planning pending updates", "Planning updates", 5)
+ s.appendAutoUpdateActivityMessage(ctx, activityID, "Planning pending updates", "Planning updates", 5)
if len(options.ResourceIds) > 0 {
- if applyErr := s.applyScopedUpdatesInternal(runCtx, options, out); applyErr != nil {
- return out, applyErr
+ if applyErr := s.applyScopedUpdates(runCtx, options, out); applyErr != nil {
+ return applyErr
}
} else {
- moduleResult, engineErr := s.engineInternal().ApplyPending(runCtx, moduleOptionsFromUpdaterOptionsInternal(options))
+ moduleResult, engineErr := s.engine.ApplyPending(runCtx, updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun})
if moduleResult != nil {
- out = resultFromModuleInternal(moduleResult)
- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.logResultItemsInternal(ctx, out)
- }
- if moduleResult == nil && engineErr == nil {
+ *out = *resultFromModule(moduleResult)
+ s.logResultItems(ctx, out)
+ } else if engineErr == nil {
engineErr = errors.New("updater returned no batch result")
}
if engineErr != nil {
- err = engineErr
- return out, err
+ return engineErr
}
}
if !options.DryRun && s.deps.ImageUpdates != nil {
- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)
+ s.appendAutoUpdateActivityMessage(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)
if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(runCtx); cleanupErr != nil {
- s.loggerInternal().WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)
+ slog.WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)
}
}
- s.recordAutoUpdateEventInternal(ctx, event.EventSeverityInfo, database.JSON{
+ s.recordAutoUpdateEvent(ctx, event.EventSeverityInfo, database.JSON{
"phase": "complete",
"checked": out.Checked,
"updated": out.Updated,
@@ -317,15 +358,60 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater
"duration": out.Duration,
"time": time.Now().UTC().Format(time.RFC3339),
})
- batchCompleted = true
- return out, nil
+ return nil
+}
+
+// scopedContainerIDs resolves the requested containers, projects, or images to container IDs.
+func (s *UpdaterService) scopedContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, error) {
+ requested := kit.TrimNonEmpty(options.ResourceIds)
+ scope := strings.ToLower(strings.TrimSpace(options.Type))
+ containerIDs := requested
+ switch {
+ case len(requested) == 0, scope == "", scope == "container":
+ case scope == "project", scope == "image":
+ // Projects match by lowercase Compose project name; images by ID or normalized reference.
+ wanted := make(map[string]bool, len(requested)*2)
+ for _, ref := range requested {
+ if scope == "image" {
+ wanted[ref], wanted[refs.NormalizeImageUpdateRef(ref)] = true, true
+ continue
+ }
+ name, discovered := strings.CutPrefix(ref, "compose:")
+ if !discovered && s.deps.Projects != nil {
+ if project, lookupErr := s.deps.Projects.GetProjectFromDatabaseByID(ctx, ref); lookupErr == nil && project != nil {
+ name = cmp.Or(strings.TrimSpace(kit.FromPtr(project.ComposeProjectName)), strings.TrimSpace(project.Name), name)
+ }
+ }
+ wanted[strings.ToLower(strings.TrimSpace(name))] = true
+ }
+ delete(wanted, "")
+ if s.deps.Docker == nil {
+ return nil, errors.New("docker service unavailable")
+ }
+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list containers: %w", err)
+ }
+ containerIDs = nil
+ for _, summary := range containers {
+ matched := wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(summary.Image)]
+ if scope == "project" {
+ project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))
+ matched = docker.ComposeServiceLabel(summary.Labels) != "" && wanted[project]
+ }
+ if matched {
+ containerIDs = append(containerIDs, summary.ID)
+ }
+ }
+ default:
+ return nil, fmt.Errorf("unsupported scoped update type %q", options.Type)
+ }
+ return containerIDs, nil
}
-// applyScopedUpdatesInternal runs a scoped update into the caller's result:
-// resolves the requested resources to container IDs and updates each through
-// the engine's single-container path.
-func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {
- containerIDs, err := s.resolveScopedContainerIDsInternal(ctx, options)
+// applyScopedUpdates updates each requested container through the engine's single-container path.
+func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {
+ containerIDs, err := s.scopedContainerIDs(ctx, options)
if err != nil {
return err
}
@@ -333,17 +419,17 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options
return common.ErrUpdaterNoContainersMatched
}
- engineOpts := moduleOptionsFromUpdaterOptionsInternal(options)
+ engineOpts := updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun}
var engineErrs []error
for _, containerID := range containerIDs {
target := scheduler.TargetOutcome{ID: containerID, ResourceType: "container", Status: scheduler.Running}
if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
return progressErr
}
- moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, engineOpts)
+ moduleResult, engineErr := s.engine.UpdateContainer(ctx, containerID, engineOpts)
target.Status = scheduler.NeedsAttention
if moduleResult != nil {
- partial := resultFromModuleInternal(moduleResult)
+ partial := resultFromModule(moduleResult)
out.Checked += partial.Checked
out.Updated += partial.Updated
out.Restarted += partial.Restarted
@@ -369,205 +455,94 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options
target.Status = scheduler.Failed
target.Message = engineErr.Error()
}
- if progressErr2 := jobcontext.Progress(ctx, target); progressErr2 != nil {
- return errors.Join(progressErr2, errors.Join(engineErrs...))
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return errors.Join(progressErr, errors.Join(engineErrs...))
}
}
- s.logResultItemsInternal(ctx, out)
+ s.logResultItems(ctx, out)
out.Success = out.Failed == 0
- // Engine errors propagate like the unscoped path's engine error does —
- // the remaining containers were still attempted and recorded above.
+ // Like the unscoped engine error, these surface after every container was attempted.
return errors.Join(engineErrs...)
}
-// resolveScopedContainerIDsInternal maps a scoped options payload to the
-// container IDs it covers.
-func (s *UpdaterService) resolveScopedContainerIDsInternal(ctx context.Context, options arcaneupdater.Options) ([]string, error) {
- requested := kit.TrimNonEmpty(options.ResourceIds)
- if len(requested) == 0 {
- return nil, nil
- }
-
- switch strings.ToLower(strings.TrimSpace(options.Type)) {
- case "", "container":
- return requested, nil
- case "project":
- return s.containerIDsForProjectsInternal(ctx, requested)
- case "image":
- return s.containerIDsForImagesInternal(ctx, requested)
- default:
- return nil, fmt.Errorf("unsupported scoped update type %q", options.Type)
- }
-}
-
-// containerIDsForProjectsInternal resolves project IDs or compose names to
-// the IDs of the containers that belong to those projects.
-func (s *UpdaterService) containerIDsForProjectsInternal(ctx context.Context, projectRefs []string) ([]string, error) {
- if s.deps.Docker == nil {
- return nil, errors.New("docker service unavailable")
- }
-
- names := make(map[string]struct{}, len(projectRefs))
- for _, ref := range projectRefs {
- name := strings.TrimSpace(ref)
- if discoveredName, discovered := strings.CutPrefix(name, "compose:"); discovered {
- name = discoveredName
- } else if s.deps.Projects != nil {
- if project, lookupErr := s.deps.Projects.GetProjectFromDatabaseByID(ctx, ref); lookupErr == nil && project != nil {
- switch {
- case project.ComposeProjectName != nil && strings.TrimSpace(*project.ComposeProjectName) != "":
- name = *project.ComposeProjectName
- case strings.TrimSpace(project.Name) != "":
- name = project.Name
- }
- }
- }
- names[strings.ToLower(strings.TrimSpace(name))] = struct{}{}
- }
-
- containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)
- if err != nil {
- return nil, fmt.Errorf("list containers: %w", err)
- }
-
- var ids []string
- for _, summary := range containers {
- if docker.ComposeServiceLabel(summary.Labels) == "" {
- continue
- }
- project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))
- if project == "" {
- continue
- }
- if _, ok := names[project]; ok {
- ids = append(ids, summary.ID)
- }
- }
- return ids, nil
-}
-
-// containerIDsForImagesInternal resolves image IDs or references to the IDs
-// of the containers currently running those images.
-func (s *UpdaterService) containerIDsForImagesInternal(ctx context.Context, imageRefs []string) ([]string, error) {
- if s.deps.Docker == nil {
- return nil, errors.New("docker service unavailable")
- }
-
- wanted := make(map[string]struct{}, len(imageRefs)*2)
- for _, ref := range imageRefs {
- trimmed := strings.TrimSpace(ref)
- if trimmed == "" {
- continue
- }
- wanted[trimmed] = struct{}{}
- if normalized := refs.NormalizeImageUpdateRef(trimmed); normalized != "" {
- wanted[normalized] = struct{}{}
- }
- }
-
- containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)
- if err != nil {
- return nil, fmt.Errorf("list containers: %w", err)
- }
-
- var ids []string
- for _, summary := range containers {
- if _, ok := wanted[summary.ImageID]; ok {
- ids = append(ids, summary.ID)
- continue
- }
- if normalized := refs.NormalizeImageUpdateRef(summary.Image); normalized != "" {
- if _, ok := wanted[normalized]; ok {
- ids = append(ids, summary.ID)
- }
- }
- }
- return ids, nil
-}
-
// UpdateSingleContainer updates a single container by ID to the latest available image.
func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID string) (out *arcaneupdater.Result, err error) {
- localActivity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, true)
- if err != nil {
- return nil, err
- }
- activityID := ""
- if localActivity != nil {
- activityID = localActivity.ID
+ activityID, workCtx := "", ctx
+ if s.deps.Activity != nil {
+ localActivity, trackedCtx, startErr := s.deps.Activity.StartTrackedActivity(ctx, s.singleContainerActivityRequest(ctx, containerID))
+ if startErr != nil {
+ return nil, fmt.Errorf("start container update activity: %w", startErr)
+ }
+ activityID, workCtx = localActivity.ID, trackedCtx
}
defer func() {
- s.finishSingleContainerUpdateInternal(workCtx, activityID, out, err)
+ s.finishSingleContainerUpdate(workCtx, activityID, out, err)
}()
defer utils.RecoverToError(&err, "single container update")
- return s.runSingleContainerUpdateInternal(workCtx, containerID, activityID)
+ return s.runSingleContainerUpdate(workCtx, containerID, activityID)
}
-// AcceptSingleContainerUpdate persists and submits a cancellable update activity.
+// AcceptSingleContainerUpdate submits a cancellable container-update workflow and returns its activity.
func (s *UpdaterService) AcceptSingleContainerUpdate(ctx context.Context, containerID string) (*activitytypes.Activity, error) {
- if s.deps.Activity == nil || !s.execution.Ready() {
+ if s.deps.Activity == nil {
return nil, errors.New("asynchronous container updates unavailable")
}
- localActivity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, false)
+ activityID, err := s.execution.Submit(ctx, containerID, s.singleContainerActivityRequest(ctx, containerID))
if err != nil {
return nil, err
}
- if submitErr := s.execution.Submit(ctx, workCtx, containerID, localActivity.ID); submitErr != nil {
- return nil, submitErr
+ detail, err := s.deps.Activity.GetActivityDetail(ctx, "0", activityID, 1)
+ if err != nil {
+ return nil, err
}
- return localActivity, nil
+ return &detail.Activity, nil
}
-func (s *UpdaterService) runSingleContainerUpdateInternal(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {
+func (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {
var release func()
- ctx, release, err = s.acquireUpdateInternal(ctx)
+ ctx, release, err = s.acquireUpdate(ctx)
if err != nil {
return nil, err
}
defer release()
start := time.Now()
- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
- ctx = contextWithActivityIDInternal(ctx, activityID)
+ out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}
+ defer func() {
+ out.Duration = cmp.Or(out.Duration, time.Since(start).String())
+ out.ActivityID = mo.EmptyableToOption(activityID).ToPointer()
+ }()
+ ctx = contextWithActivityID(ctx, activityID)
if s.deps.Activity != nil && activityID != "" {
- if awaitActivitySlotBoundedErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); awaitActivitySlotBoundedErr != nil {
- return out, awaitActivitySlotBoundedErr
+ if slotErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {
+ return out, slotErr
}
}
- defer func() {
- if out == nil {
- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}
- }
- if out.Duration == "" {
- out.Duration = time.Since(start).String()
- }
- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- }()
-
s.updateMu.Lock()
defer s.updateMu.Unlock()
- if errErr := ctx.Err(); errErr != nil {
- return out, errErr
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return out, ctxErr
}
- // The caller picked this container, so the autoUpdateExcludedContainers
- // setting does not apply: it only governs automatic and pending runs, which
- // keep skipping the container. Labels and immutable references still do.
- moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, updater.Options{IgnoreSettingsExclusions: true})
+ // The caller picked this container, so the autoUpdateExcludedContainers setting does not
+ // apply here; labels and immutable references still do.
+ moduleResult, err := s.engine.UpdateContainer(ctx, containerID, updater.Options{IgnoreSettingsExclusions: true})
if moduleResult != nil {
- out = resultFromModuleInternal(moduleResult)
- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.logResultItemsInternal(ctx, out)
+ out = resultFromModule(moduleResult)
+ s.logResultItems(ctx, out)
}
- if engineErr != nil {
- err = engineErr
- return out, err
- }
- return out, nil
+ return out, err
}
// GetStatus returns the current in-memory update activity snapshot.
func (s *UpdaterService) GetStatus() arcaneupdater.Status {
- return statusFromModuleInternal(s.engineInternal().Status())
+ status := s.engine.Status()
+ return arcaneupdater.Status{
+ UpdatingContainers: status.UpdatingContainers,
+ UpdatingProjects: status.UpdatingProjects,
+ ContainerIds: status.ContainerIDs,
+ ProjectIds: status.ProjectIDs,
+ }
}
// GetHistory returns the most recent auto-update history records, newest first.
@@ -585,8 +560,8 @@ func (s *UpdaterService) GetHistory(ctx context.Context, limit int) ([]AutoUpdat
// RestartContainersUsingOldIDs restarts containers matching old image IDs or refs.
func (s *UpdaterService) RestartContainersUsingOldIDs(ctx context.Context, oldIDToNewRef, oldRefToNewRef map[string]string) ([]arcaneupdater.ResourceResult, error) {
- results, err := s.engineInternal().RestartContainersUsingOldImages(ctx, oldIDToNewRef, oldRefToNewRef)
- return resourceResultsFromModuleInternal(results), err
+ results, err := s.engine.RestartContainersUsingOldImages(ctx, oldIDToNewRef, oldRefToNewRef)
+ return resultFromModule(&updater.Result{Items: results}).Items, err
}
// TriggerSelfUpdateViaCLI triggers Arcane's detached CLI self-update path.
@@ -604,53 +579,49 @@ func (s *UpdaterService) TriggerSelfUpdateViaCLI(ctx context.Context, source, co
// BeginContainerUpdate marks a container as updating.
func (s *UpdaterService) BeginContainerUpdate(containerID string) func() {
- return s.engineInternal().BeginContainerUpdate(containerID)
+ return s.engine.BeginContainerUpdate(containerID)
}
// BeginProjectUpdate marks a project as updating.
func (s *UpdaterService) BeginProjectUpdate(projectID string) func() {
- return s.engineInternal().BeginProjectUpdate(projectID)
+ return s.engine.BeginProjectUpdate(projectID)
}
-func (s *UpdaterService) recordAutoUpdateEventInternal(ctx context.Context, severity event.EventSeverity, metadata database.JSON) {
+func (s *UpdaterService) recordAutoUpdateEvent(ctx context.Context, severity event.EventSeverity, metadata database.JSON) {
if s.deps.Events == nil {
return
}
phase, _ := metadata["phase"].(string)
- _, err := s.deps.Events.CreateEvent(ctx, event.CreateEventRequest{
- Type: event.EventTypeSystemAutoUpdate,
- Severity: severity,
- Title: autoUpdateEventTitleInternal(phase, metadata),
- ResourceType: mo.EmptyableToOption(strings.TrimSpace("system")).ToPointer(),
- ResourceName: mo.EmptyableToOption(strings.TrimSpace("auto_updater")).ToPointer(),
- EnvironmentID: mo.EmptyableToOption(strings.TrimSpace("0")).ToPointer(),
- Metadata: metadata,
- })
- if err != nil {
- s.loggerInternal().DebugContext(ctx, "failed to record auto-update event", "error", err)
- }
-}
-
-func autoUpdateEventTitleInternal(phase string, metadata database.JSON) string {
+ title, subject := kit.Ternary(phase != "", "Auto-update: "+phase, "Auto-update"), ""
switch phase {
case "start":
- return "Auto-update run started"
+ title = "Auto-update run started"
+ case "complete":
+ title = "Auto-update run completed"
case "image_pull", "image":
- localImage := cmp.Or(kit.ToString(metadata["imageNew"]), kit.ToString(metadata["imageOld"]))
- return kit.Ternary(localImage != "", "Auto-update: image pull "+localImage, "Auto-update: image pull")
+ title, subject = "Auto-update: image pull", cmp.Or(kit.ToString(metadata["imageNew"]), kit.ToString(metadata["imageOld"]))
case "image_prune":
- imageID := kit.ToString(metadata["imageId"])
- return kit.Ternary(imageID != "", "Auto-update: image prune "+imageID, "Auto-update: image prune")
+ title, subject = "Auto-update: image prune", kit.ToString(metadata["imageId"])
case "container":
- name := cmp.Or(kit.ToString(metadata["resourceName"]), kit.ToString(metadata["container"]), kit.ToString(metadata["containerId"]))
- return kit.Ternary(name != "", "Auto-update: container "+name, "Auto-update: container")
+ title = "Auto-update: container"
+ subject = cmp.Or(kit.ToString(metadata["resourceName"]), kit.ToString(metadata["container"]), kit.ToString(metadata["containerId"]))
case "project":
- name := cmp.Or(kit.ToString(metadata["projectName"]), kit.ToString(metadata["projectId"]))
- return kit.Ternary(name != "", "Auto-update: project "+name, "Auto-update: project")
- case "complete":
- return "Auto-update run completed"
- default:
- return kit.Ternary(phase != "", "Auto-update: "+phase, "Auto-update")
+ title, subject = "Auto-update: project", cmp.Or(kit.ToString(metadata["projectName"]), kit.ToString(metadata["projectId"]))
+ }
+ if subject != "" {
+ title += " " + subject
+ }
+ _, err := s.deps.Events.CreateEvent(ctx, event.CreateEventRequest{
+ Type: event.EventTypeSystemAutoUpdate,
+ Severity: severity,
+ Title: title,
+ ResourceType: new("system"),
+ ResourceName: new("auto_updater"),
+ EnvironmentID: new("0"),
+ Metadata: metadata,
+ })
+ if err != nil {
+ slog.DebugContext(ctx, "failed to record auto-update event", "error", err)
}
}
@@ -662,10 +633,8 @@ func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, erro
return s.deps.Docker.GetClient(ctx)
}
-// PullImage pulls an image through Arcane's image service. The pull is
-// bounded by the dockerImagePullTimeout setting — image.ImageService.PullImage does
-// not bound itself, and an unbounded engine pull would hold the auto-update
-// run (and its activity slot) indefinitely.
+// PullImage pulls an image through Arcane's image service, bounded by the
+// dockerImagePullTimeout setting so a stuck pull cannot hold the auto-update run.
func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {
if s == nil || s.deps.ImagePuller == nil {
return common.Classify(common.ErrUnavailable, errors.New("image service unavailable"))
@@ -674,8 +643,7 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
if err != nil {
return err
}
- activityID := activityIDFromContextInternal(ctx)
- writer := activitylib.NewWriter(ctx, s.deps.Activity, activityID, progress, "Pulling updated images")
+ writer := activitylib.NewWriter(ctx, s.deps.Activity, activityIDFromContext(ctx), progress, "Pulling updated images")
defer activitylib.FlushWriter(writer)
pullTimeoutSeconds := 0
@@ -687,22 +655,24 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
var credentials []containerregistry.Credential
if s.deps.Projects != nil {
- resolved, resolveRegistryCredentialsErr := s.deps.Projects.ResolveRegistryCredentials(pullCtx)
- if resolveRegistryCredentialsErr != nil {
- return fmt.Errorf("resolve registry credentials: %w", resolveRegistryCredentialsErr)
+ if credentials, err = s.deps.Projects.ResolveRegistryCredentials(pullCtx); err != nil {
+ return fmt.Errorf("resolve registry credentials: %w", err)
}
- credentials = resolved
}
- if pullImageErr := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, credentials); pullImageErr != nil {
- return pullImageErr
+ if pullErr := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, credentials); pullErr != nil {
+ return pullErr
}
- if tagErr := s.tagFrozenPullInternal(pullCtx, pulledRef, imageRef); tagErr != nil {
- return tagErr
+ // A frozen digest pull is tagged back to the reference the engine asked for.
+ if pulledRef != imageRef {
+ dockerClient, clientErr := s.DockerClient(pullCtx)
+ if clientErr != nil {
+ return clientErr
+ }
+ if _, tagErr := dockerClient.ImageTag(pullCtx, client.ImageTagOptions{Source: pulledRef, Target: imageRef}); tagErr != nil {
+ return tagErr
+ }
}
- // Reconcile by tag so the new image's record and any project preview rows
- // for this reference stop reporting the update just installed (#4306).
- // This runs on the run context, not the pull deadline, so a pull that
- // finished late still records its result.
+ // Reconcile by tag on the run context so a late pull still clears its update records (#4306).
if reconcileErr := s.deps.ImagePuller.ReconcilePulledImageUpdate(ctx, imageRef); reconcileErr != nil {
slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", reconcileErr)
}
@@ -723,24 +693,62 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Ima
return nil, fmt.Errorf("query pending image updates: %w", err)
}
- // Flush pending "Updates Available" notifications before the engine
- // consumes and clears these records, otherwise the notification for an
- // update applied here is silently lost (#3132). A flush that cannot
- // determine what to send aborts the run so the records survive for retry.
+ // Flush "Updates Available" notifications before the engine clears these records (#3132);
+ // a flush that cannot decide what to send aborts so the records survive for retry.
if s.deps.ImageUpdates != nil {
if err := s.deps.ImageUpdates.SendBatchUpdateNotifications(ctx); err != nil {
return nil, err
}
}
- s.appendAutoUpdateActivityMessageInternal(
- ctx,
- activityIDFromContextInternal(ctx),
- fmt.Sprintf("Found %d pending image update records", len(records)),
- "Planning updates",
- 10,
- )
+ s.appendAutoUpdateActivityMessage(ctx, activityIDFromContext(ctx), fmt.Sprintf("Found %d pending image update records", len(records)), "Planning updates", 10)
- return s.scopedPendingRecordsInternal(ctx, records)
+ // Once container-scoped records exist, an unscoped record applies per running non-tag-policy container of its image.
+ var scopedCount int64
+ if err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {
+ return nil, err
+ }
+ var running []container.Summary
+ if scopedCount > 0 {
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ if err != nil {
+ return nil, err
+ }
+ running = listed.Items
+ }
+ out := make([]updater.ImageUpdateRecord, 0, len(records))
+ for _, record := range records {
+ converted := updater.ImageUpdateRecord{
+ ID: record.ID,
+ ContainerID: record.ContainerID,
+ Repository: record.Repository,
+ Tag: record.Tag,
+ HasUpdate: record.HasUpdate,
+ UpdateType: updater.UpdateType(record.UpdateType),
+ CurrentVersion: record.CurrentVersion,
+ LatestVersion: record.LatestVersion,
+ CurrentDigest: record.CurrentDigest,
+ LatestDigest: record.LatestDigest,
+ CheckTime: record.CheckTime,
+ LastError: record.LastError,
+ }
+ if scopedCount == 0 || record.ContainerID != "" {
+ out = append(out, converted)
+ continue
+ }
+ for _, cnt := range running {
+ resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
+ if policyErr != nil || resolved.Strategy == "tag" || refs.NormalizeImageUpdateRef(cnt.Image) != refs.NormalizeImageUpdateRef(converted.ImageRef()) {
+ continue
+ }
+ converted.ContainerID = cnt.ID
+ out = append(out, converted)
+ }
+ }
+ return out, nil
}
// ClearImageUpdateRecord clears a pending image update record after it is handled.
@@ -748,15 +756,46 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record upda
if s == nil {
return common.Classify(common.ErrUnavailable, errors.New("updater service unavailable"))
}
- return s.clearImageUpdateRecordForModuleInternal(ctx, record)
+ if s.deps.DB == nil {
+ return nil
+ }
+ query := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{})
+ if record.ContainerID == "" || strings.HasPrefix(record.ID, "container::") {
+ query = query.Where("container_id = ?", record.ContainerID)
+ if record.ID != "" {
+ return query.Where("id = ?", record.ID).Update("has_update", false).Error
+ }
+ return query.Where("repository = ? AND tag = ?", record.Repository, record.Tag).Update("has_update", false).Error
+ }
+
+ // A shared record stays pending while another non-tag-policy container still runs an older image.
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return err
+ }
+ target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())
+ if err != nil {
+ return err
+ }
+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ if err != nil {
+ return err
+ }
+ for _, cnt := range listed.Items {
+ resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
+ if policyErr != nil || resolved.Strategy == "tag" {
+ continue
+ }
+ if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {
+ return nil
+ }
+ }
+ return query.Where("id = ? AND container_id = ?", record.ID, "").Update("has_update", false).Error
}
// ExcludedContainers returns auto-update exclusions from Arcane settings.
func (s *UpdaterService) ExcludedContainers(ctx context.Context) ([]string, error) {
- if s == nil {
- return nil, nil
- }
- if s.deps.Settings == nil {
+ if s == nil || s.deps.Settings == nil {
return nil, nil
}
return kit.Unique(kit.TrimNonEmpty(strings.Split(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ","))), nil
@@ -801,17 +840,23 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.S
return fmt.Errorf("%s self-update requires CLI upgrade service", instanceType)
}
- // A server self-update stops this process before the run can complete its
- // activity, so annotate the activity first; startup reconciliation uses
- // the metadata flag to finalize it after the restart.
- if target.InstanceType != "agent" {
- s.markSelfUpdateTriggeredInternal(ctx, target.NewImageRef)
+ // A server self-update stops this process before the run completes, so the
+ // activity is flagged first and startup reconciliation finalizes it.
+ if activityID := activityIDFromContext(ctx); target.InstanceType != "agent" && activityID != "" && s.deps.Activity != nil {
+ message := "Self-update initiated — Arcane will restart"
+ if target.NewImageRef != "" {
+ message += " with " + target.NewImageRef
+ }
+ s.appendAutoUpdateActivityMessage(ctx, activityID, message, "Self-update", 90)
+ if err := s.deps.Activity.PatchActivityMetadata(ctx, activityID, database.JSON{"selfUpdateTriggered": true}); err != nil {
+ slog.DebugContext(ctx, "failed to mark self-update on activity", "activityId", activityID, "error", err)
+ }
}
if _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {
return fmt.Errorf("CLI upgrade failed: %w", err)
}
- if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok {
+ if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok {
progress.mu.Lock()
progress.selfTriggered = true
progress.selfID = target.ContainerID
@@ -820,30 +865,13 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.S
return nil
}
-func (s *UpdaterService) markSelfUpdateTriggeredInternal(ctx context.Context, imageRef string) {
- activityID := activityIDFromContextInternal(ctx)
- if s.deps.Activity == nil || activityID == "" {
- return
- }
- message := "Self-update initiated — Arcane will restart"
- if ref := strings.TrimSpace(imageRef); ref != "" {
- message = "Self-update initiated — Arcane will restart with " + ref
- }
- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, message, "Self-update", 90)
- if err := s.deps.Activity.PatchActivityMetadata(ctx, activityID, database.JSON{"selfUpdateTriggered": true}); err != nil {
- slog.DebugContext(ctx, "failed to mark self-update on activity", "activityId", activityID, "error", err)
- }
-}
-
-// Notify buffers Arcane's container update notification when called within an
-// auto-update run (see withBatchedNotificationsInternal); buffered entries are
-// flushed as one batched notification when the run completes. Outside a run it
-// sends the legacy per-container notification immediately.
+// Notify buffers the container update notification inside an update batch, which sends
+// them as one notification when it ends; outside a batch it sends immediately.
func (s *UpdaterService) Notify(ctx context.Context, localNotification updater.Notification) error {
if s == nil || s.deps.Notifications == nil {
return nil
}
- if buffer := batchedContainerUpdatesFromContextInternal(ctx); buffer != nil {
+ if buffer, ok := ctx.Value(containerUpdateBatchContextKey{}).(*containerUpdateBatch); ok {
buffer.Lock()
buffer.entries = append(buffer.entries, notifications.ContainerUpdateBatchEntry{
ContainerName: localNotification.ContainerName,
@@ -863,47 +891,13 @@ func (s *UpdaterService) Notify(ctx context.Context, localNotification updater.N
)
}
-// containerUpdateBatchInternal accumulates per-container update notifications
-// so a single auto-update run produces one batched notification.
-type containerUpdateBatchInternal struct {
- sync.Mutex
-
- entries []notifications.ContainerUpdateBatchEntry
-}
-
-type containerUpdateBatchContextKeyInternal struct{}
-
-func batchedContainerUpdatesFromContextInternal(ctx context.Context) *containerUpdateBatchInternal {
- batch, _ := ctx.Value(containerUpdateBatchContextKeyInternal{}).(*containerUpdateBatchInternal)
- return batch
-}
-
-// flushBatchedContainerUpdatesInternal delivers the accumulated container
-// update notifications as one batched notification.
-func (s *UpdaterService) flushBatchedContainerUpdatesInternal(ctx context.Context, batch *containerUpdateBatchInternal) {
- if s == nil || s.deps.Notifications == nil || batch == nil {
- return
- }
- batch.Lock()
- entries := batch.entries
- batch.entries = nil
- batch.Unlock()
- if len(entries) == 0 {
- return
- }
- if err := s.deps.Notifications.SendBatchContainerUpdateNotification(ctx, entries); err != nil {
- s.loggerInternal().ErrorContext(ctx, "failed to send batched container update notification", "error", err, "count", len(entries))
- }
-}
-
// RecordEvent records updater lifecycle events in Arcane's event stream.
func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) error {
if s == nil {
return nil
}
- eventType, ok := containerEventTypeInternal(evt.Phase).Get()
- if ok {
+ if eventType, ok := containerEventTypes[evt.Phase]; ok {
if s.deps.Events == nil {
return nil
}
@@ -920,7 +914,7 @@ func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) err
}
severity := kit.Ternary(strings.EqualFold(evt.Severity, "error"), event.EventSeverityError, event.EventSeverityInfo)
- s.recordAutoUpdateEventInternal(ctx, severity, database.JSON{
+ s.recordAutoUpdateEvent(ctx, severity, database.JSON{
"phase": evt.Phase,
"resourceId": evt.ResourceID,
"resourceName": evt.ResourceName,
@@ -930,73 +924,28 @@ func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) err
return nil
}
-func containerEventTypeInternal(phase string) mo.Option[event.EventType] {
- switch phase {
- case "container_stop":
- return mo.Some(event.EventTypeContainerStop)
- case "container_delete":
- return mo.Some(event.EventTypeContainerDelete)
- case "container_create":
- return mo.Some(event.EventTypeContainerCreate)
- case "container_start":
- return mo.Some(event.EventTypeContainerStart)
- case "container_update":
- return mo.Some(event.EventTypeContainerUpdate)
- default:
- return mo.None[event.EventType]()
- }
-}
-
-type activityIDContextKeyInternal struct{}
-
-func contextWithActivityIDInternal(ctx context.Context, activityID string) context.Context {
- activityID = strings.TrimSpace(activityID)
+func contextWithActivityID(ctx context.Context, activityID string) context.Context {
if activityID == "" {
return ctx
}
- return context.WithValue(ctx, activityIDContextKeyInternal{}, activityID)
+ return context.WithValue(ctx, activityIDContextKey{}, activityID)
}
-func activityIDFromContextInternal(ctx context.Context) string {
+func activityIDFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
- activityID, _ := ctx.Value(activityIDContextKeyInternal{}).(string)
- return strings.TrimSpace(activityID)
+ activityID, _ := ctx.Value(activityIDContextKey{}).(string)
+ return activityID
}
-func (s *UpdaterService) startAutoUpdateActivityInternal(ctx context.Context, dryRun bool) string {
- if s.deps.Activity == nil {
- return ""
- }
- localActivity, err := s.deps.Activity.StartActivity(ctx, activitylib.StartRequest{
- EnvironmentID: "0",
- Type: activitytypes.TypeAutoUpdate,
- Queue: true,
- ResourceType: mo.EmptyableToOption(strings.TrimSpace("system")).ToPointer(),
- ResourceName: mo.EmptyableToOption(strings.TrimSpace("Auto update")).ToPointer(),
- Step: "Planning updates",
- LatestMessage: "Auto-update run started",
- Metadata: database.JSON{"dryRun": dryRun},
- })
- if err != nil {
- slog.DebugContext(ctx, "failed to start auto-update activity", "error", err)
- return ""
- }
- return localActivity.ID
-}
-
-func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.Context, containerID string, tracked bool) (*activitytypes.Activity, context.Context, error) {
- if s.deps.Activity == nil {
- return nil, ctx, nil
- }
+// singleContainerActivityRequest describes a container update's activity, named after the container when it can be inspected.
+func (s *UpdaterService) singleContainerActivityRequest(ctx context.Context, containerID string) activitylib.StartRequest {
name := containerID
lookupCtx, cancelLookup := context.WithTimeout(ctx, 2*time.Second)
- if dockerClient, dockerErr := s.DockerClient(lookupCtx); dockerErr == nil && dockerClient != nil {
+ if dockerClient, dockerErr := s.DockerClient(lookupCtx); dockerErr == nil {
if inspected, inspectErr := compat.ContainerInspectWithCompatibility(lookupCtx, dockerClient, containerID, client.ContainerInspectOptions{}); inspectErr == nil {
- if actualName := strings.TrimPrefix(strings.TrimSpace(inspected.Container.Name), "/"); actualName != "" {
- name = actualName
- }
+ name = cmp.Or(strings.TrimPrefix(inspected.Container.Name, "/"), containerID)
}
}
cancelLookup()
@@ -1008,12 +957,12 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.
user.DisplayName = &initiator.DisplayName
}
}
- request := activitylib.StartRequest{
+ return activitylib.StartRequest{
EnvironmentID: "0",
Type: activitytypes.TypeAutoUpdate,
Queue: true,
DeferSlot: true,
- ResourceType: mo.EmptyableToOption(strings.TrimSpace("container")).ToPointer(),
+ ResourceType: new("container"),
ResourceID: &containerID,
ResourceName: &name,
StartedBy: user,
@@ -1021,92 +970,72 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.
LatestMessage: "Container update started",
Metadata: database.JSON{"containerID": containerID},
}
- var item *activitytypes.Activity
- workCtx := ctx
- var err error
- if tracked {
- item, workCtx, err = s.deps.Activity.StartTrackedActivity(ctx, request)
- } else {
- item, err = s.deps.Activity.StartActivity(ctx, request)
- }
- if err != nil {
- return nil, nil, fmt.Errorf("start container update activity: %w", err)
- }
- return item, workCtx, nil
}
-func (s *UpdaterService) finishSingleContainerUpdateInternal(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) {
+func (s *UpdaterService) finishSingleContainerUpdate(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) {
if s.deps.Activity == nil || activityID == "" {
return
}
- metadata, message := singleContainerActivitySummaryInternal(result)
- writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- defer cancel()
- if containerName, ok := metadata["containerName"].(string); ok && containerName != "" {
- if _, err := s.deps.Activity.UpdateActivity(writeCtx, activityID, activitylib.UpdateRequest{ResourceName: &containerName}); err != nil {
- slog.WarnContext(writeCtx, "failed to update container activity name", "activityId", activityID, "error", err)
+ outcome := s.recordSingleContainerUpdate(ctx, activityID, result, runErr)
+ s.completeAutoUpdateActivity(ctx, activityID, outcome.Message, kit.Ternary(outcome.Status != scheduler.Succeeded, outcome.Message, ""))
+}
+
+// recordSingleContainerUpdate saves a container update's result on its activity and summarizes it as an outcome.
+func (s *UpdaterService) recordSingleContainerUpdate(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) scheduler.Outcome {
+ metadata, containerName := database.JSON{}, ""
+ if result != nil {
+ metadata = database.JSON{"updated": result.Updated, "restarted": result.Restarted, "skipped": result.Skipped, "failed": result.Failed}
+ if len(result.Items) > 0 {
+ item := result.Items[0]
+ metadata["updateOutcome"] = item.Status
+ if containerName = item.ResourceName; containerName != "" {
+ metadata["containerName"] = containerName
+ }
+ if item.Error != "" {
+ metadata["updateReason"] = item.Error
+ }
}
}
- if err := s.deps.Activity.PatchActivityMetadata(writeCtx, activityID, metadata); err != nil {
- slog.WarnContext(writeCtx, "failed to persist container update result", "activityId", activityID, "error", err)
+ if s.deps.Activity != nil && activityID != "" {
+ writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ if containerName != "" {
+ if _, err := s.deps.Activity.UpdateActivity(writeCtx, activityID, activitylib.UpdateRequest{ResourceName: &containerName}); err != nil {
+ slog.WarnContext(writeCtx, "failed to update container activity name", "activityId", activityID, "error", err)
+ }
+ }
+ if err := s.deps.Activity.PatchActivityMetadata(writeCtx, activityID, metadata); err != nil {
+ slog.WarnContext(writeCtx, "failed to persist container update result", "activityId", activityID, "error", err)
+ }
}
if runErr == nil && result != nil && result.Failed == 0 {
- if result.Updated > 0 || result.Restarted > 0 {
+ message := "Container already current"
+ switch {
+ case result.Updated > 0 || result.Restarted > 0:
message = "Container updated"
+ case len(result.Items) > 0 && result.Items[0].Status == string(updater.StatusSkipped) && result.Items[0].Error != "":
+ message = "Container update skipped: " + result.Items[0].Error
+ case result.Skipped > 0:
+ message = "Container update skipped"
}
- if _, err := s.deps.Activity.CompleteActivity(writeCtx, activityID, activitytypes.StatusSuccess, message, nil); err != nil {
- slog.ErrorContext(writeCtx, "failed to complete container update activity", "activityId", activityID, "error", err)
- }
- return
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: message}
}
- if runErr == nil && result == nil {
+ switch {
+ case runErr != nil:
+ case result == nil:
runErr = errors.New("container update produced no result")
- }
- if runErr == nil && result != nil && len(result.Items) > 0 && result.Items[0].Error != "" {
+ case len(result.Items) > 0 && result.Items[0].Error != "":
runErr = errors.New(result.Items[0].Error)
+ default:
+ runErr = fmt.Errorf("%d update action(s) failed", result.Failed)
}
- s.completeAutoUpdateActivityInternal(ctx, activityID, result, runErr)
-}
-
-func singleContainerActivitySummaryInternal(result *arcaneupdater.Result) (database.JSON, string) {
- metadata := database.JSON{}
- message := "Container update completed"
- if result == nil {
- return metadata, message
- }
- metadata["updated"] = result.Updated
- metadata["restarted"] = result.Restarted
- metadata["skipped"] = result.Skipped
- metadata["failed"] = result.Failed
- if len(result.Items) > 0 {
- item := result.Items[0]
- metadata["updateOutcome"] = item.Status
- if item.ResourceName != "" {
- metadata["containerName"] = item.ResourceName
- }
- if item.Error != "" {
- metadata["updateReason"] = item.Error
- if item.Status == string(updater.StatusSkipped) {
- return metadata, "Container update skipped: " + item.Error
- }
- }
- }
- if result.Skipped > 0 {
- return metadata, "Container update skipped"
- }
- if result.Updated == 0 && result.Restarted == 0 && result.Failed == 0 {
- message = "Container already current"
- }
- return metadata, message
+ return scheduler.Outcome{Status: scheduler.Failed, Message: runErr.Error()}
}
-func (s *UpdaterService) appendAutoUpdateActivityMessageInternal(ctx context.Context, activityID, message, step string, progress int) {
- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {
+func (s *UpdaterService) appendAutoUpdateActivityMessage(ctx context.Context, activityID, message, step string, progress int) {
+ if s.deps.Activity == nil || activityID == "" {
return
}
- if strings.TrimSpace(step) == "" {
- step = message
- }
if _, err := s.deps.Activity.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{
Level: activitytypes.MessageLevelInfo,
Message: message,
@@ -1117,207 +1046,69 @@ func (s *UpdaterService) appendAutoUpdateActivityMessageInternal(ctx context.Con
}
}
-func (s *UpdaterService) completeAutoUpdateActivityInternal(ctx context.Context, activityID string, result *arcaneupdater.Result, applyErr error) {
- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {
+// completeAutoUpdateActivity ends the activity with message, or as failed when failure is set;
+// a failure caused by cancelling the activity ends it as cancelled.
+func (s *UpdaterService) completeAutoUpdateActivity(ctx context.Context, activityID, message, failure string) {
+ if s.deps.Activity == nil || activityID == "" {
return
}
-
status := activitytypes.StatusSuccess
- message := "Auto-update run completed"
var errMessage *string
- if applyErr != nil {
- status = activitytypes.StatusFailed
- errText := applyErr.Error()
- errMessage = &errText
- message = errText
- } else if result != nil && result.Failed > 0 {
- status = activitytypes.StatusFailed
- errText := fmt.Sprintf("%d update action(s) failed", result.Failed)
- errMessage = &errText
- message = errText
- }
- if status == activitytypes.StatusFailed && activitylib.CancelledByContext(ctx) {
- status = activitytypes.StatusCancelled
- message = "Auto-update cancelled"
- errMessage = nil
- }
-
- if _, err := s.deps.Activity.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {
- // A lost terminal write strands the activity in running forever, so it
- // must be loud enough to correlate with a stuck activity panel entry.
- slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)
- }
-}
-
-func (s *UpdaterService) trackActivityInternal(ctx context.Context, activityID string) context.Context {
- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {
- return ctx
- }
- return s.deps.Activity.Track(ctx, activityID)
-}
-
-func imageUpdateRecordToModuleInternal(record imageupdate.ImageUpdateRecord) updater.ImageUpdateRecord {
- return updater.ImageUpdateRecord{
- ID: record.ID,
- ContainerID: record.ContainerID,
- Repository: record.Repository,
- Tag: record.Tag,
- HasUpdate: record.HasUpdate,
- UpdateType: updater.UpdateType(record.UpdateType),
- CurrentVersion: record.CurrentVersion,
- LatestVersion: record.LatestVersion,
- CurrentDigest: record.CurrentDigest,
- LatestDigest: record.LatestDigest,
- CheckTime: record.CheckTime,
- LastError: record.LastError,
+ switch {
+ case failure == "":
+ case activitylib.CancelledByContext(ctx):
+ status, message = activitytypes.StatusCancelled, "Auto-update cancelled"
+ default:
+ status, message, errMessage = activitytypes.StatusFailed, failure, &failure
}
-}
-
-// moduleOptionsFromUpdaterOptionsInternal narrows Arcane's request options to
-// what the engine acts on. Options.Type and Options.ResourceIds stay behind:
-// the engine never read them, and ApplyPending already routes a scoped request
-// through applyScopedUpdatesInternal before reaching the engine. Settings
-// exclusions always apply to these runs; only UpdateSingleContainer overrides
-// them for its explicitly requested container.
-func moduleOptionsFromUpdaterOptionsInternal(options arcaneupdater.Options) updater.Options {
- return updater.Options{
- Force: options.ForceUpdate,
- DryRun: options.DryRun,
+ // A lost terminal write strands the activity as running, so it is written past cancellation and logged loudly.
+ if _, err := s.deps.Activity.CompleteActivity(context.WithoutCancel(ctx), activityID, status, message, errMessage); err != nil {
+ slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)
}
}
-// resultFromModuleInternal converts an engine result to Arcane's wire type. The
-// engine reports times as time.Time and a Duration method; Arcane's API has
-// always carried them as strings, so they are formatted here. ActivityID is not
-// an engine concept; every caller sets it on the returned value.
-func resultFromModuleInternal(result *updater.Result) *arcaneupdater.Result {
+// resultFromModule converts an engine result to Arcane's wire type, which carries times as
+// strings and images as maps holding only the "main" entry. Callers set ActivityID.
+func resultFromModule(result *updater.Result) *arcaneupdater.Result {
if result == nil {
return &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}
}
- return &arcaneupdater.Result{
+ mainImage := func(ref string) map[string]string {
+ if ref == "" {
+ return nil
+ }
+ return map[string]string{"main": ref}
+ }
+ out := &arcaneupdater.Result{
Success: result.Success,
Checked: result.Checked,
Updated: result.Updated,
Restarted: result.Restarted,
Skipped: result.Skipped,
Failed: result.Failed,
- StartTime: formatModuleTimeInternal(result.StartTime),
- EndTime: formatModuleTimeInternal(result.EndTime),
+ StartTime: kit.Ternary(result.StartTime.IsZero(), "", result.StartTime.UTC().Format(time.RFC3339)),
+ EndTime: kit.Ternary(result.EndTime.IsZero(), "", result.EndTime.UTC().Format(time.RFC3339)),
Duration: result.Duration().String(),
- Items: resourceResultsFromModuleInternal(result.Items),
- }
-}
-
-func formatModuleTimeInternal(value time.Time) string {
- if value.IsZero() {
- return ""
- }
- return value.UTC().Format(time.RFC3339)
-}
-
-func resourceResultsFromModuleInternal(results []updater.ResourceResult) []arcaneupdater.ResourceResult {
- out := make([]arcaneupdater.ResourceResult, 0, len(results))
- for _, result := range results {
- out = append(out, resourceResultFromModuleInternal(result))
- }
- return out
-}
-
-// resourceResultFromModuleInternal converts one engine result to Arcane's wire
-// type. The engine now reports a single old/new image; Arcane's API carries
-// maps, which only ever held the "main" entry, so that shape is rebuilt here.
-func resourceResultFromModuleInternal(result updater.ResourceResult) arcaneupdater.ResourceResult {
- mainImage := func(ref string) map[string]string {
- if ref == "" {
- return nil
- }
- return map[string]string{"main": ref}
- }
- return arcaneupdater.ResourceResult{
- ResourceID: result.ResourceID,
- ResourceName: result.ResourceName,
- ResourceType: string(result.ResourceType),
- Status: string(result.Status),
- UpdateAvailable: result.UpdateAvailable,
- UpdateApplied: result.UpdateApplied,
- OldImages: mainImage(result.OldImage),
- NewImages: mainImage(result.NewImage),
- Error: result.Error,
- Details: result.Details,
- }
-}
-
-func statusFromModuleInternal(status updater.Status) arcaneupdater.Status {
- return arcaneupdater.Status{
- UpdatingContainers: status.UpdatingContainers,
- UpdatingProjects: status.UpdatingProjects,
- ContainerIds: status.ContainerIDs,
- ProjectIds: status.ProjectIDs,
- }
-}
-
-func (s *UpdaterService) recordRunInternal(ctx context.Context, item arcaneupdater.ResourceResult) error {
- now := time.Now()
- record := &AutoUpdateRecord{
- ResourceID: item.ResourceID,
- ResourceType: item.ResourceType,
- ResourceName: item.ResourceName,
- Status: AutoUpdateStatus(item.Status),
- StartTime: now,
- EndTime: &now,
- UpdateAvailable: item.UpdateAvailable || item.Status == string(updater.StatusUpdated) || item.Status == string(updater.StatusUpdateAvailable),
- UpdateApplied: item.UpdateApplied,
- OldImageVersions: mapToJSONInternal(item.OldImages),
- NewImageVersions: mapToJSONInternal(item.NewImages),
- Details: detailsToJSONInternal(item.Details),
- }
- if item.Error != "" {
- record.Error = &item.Error
- }
- return s.deps.DB.WithContext(ctx).Create(record).Error
-}
-
-func (s *UpdaterService) clearImageUpdateRecordForModuleInternal(ctx context.Context, record updater.ImageUpdateRecord) error {
- if s.deps.DB == nil {
- return nil
- }
-
- if record.ContainerID != "" && !strings.HasPrefix(record.ID, "container::") {
- return s.clearUnscopedRecordInternal(ctx, record)
- }
- query := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{})
- if record.ContainerID != "" {
- query = query.Where("container_id = ?", record.ContainerID)
- } else {
- query = query.Where("container_id = ?", "")
- }
- if strings.TrimSpace(record.ID) != "" {
- return query.Where("id = ?", record.ID).Update("has_update", false).Error
- }
- return query.Where("repository = ? AND tag = ?", record.Repository, record.Tag).Update("has_update", false).Error
-}
-
-func mapToJSONInternal(values map[string]string) database.JSON {
- if len(values) == 0 {
- return nil
+ Items: make([]arcaneupdater.ResourceResult, 0, len(result.Items)),
}
- out := make(database.JSON, len(values))
- for key, value := range values {
- out[key] = value
- }
- return out
-}
-
-func detailsToJSONInternal(values map[string]any) database.JSON {
- if len(values) == 0 {
- return nil
+ for _, item := range result.Items {
+ out.Items = append(out.Items, arcaneupdater.ResourceResult{
+ ResourceID: item.ResourceID,
+ ResourceName: item.ResourceName,
+ ResourceType: string(item.ResourceType),
+ Status: string(item.Status),
+ UpdateAvailable: item.UpdateAvailable,
+ UpdateApplied: item.UpdateApplied,
+ OldImages: mainImage(item.OldImage),
+ NewImages: mainImage(item.NewImage),
+ Error: item.Error,
+ Details: item.Details,
+ })
}
- out := make(database.JSON, len(values))
- maps.Copy(out, values)
return out
}
-func (s *UpdaterService) logResultItemsInternal(ctx context.Context, result *arcaneupdater.Result) {
+func (s *UpdaterService) logResultItems(ctx context.Context, result *arcaneupdater.Result) {
if result == nil {
return
}
@@ -1329,7 +1120,7 @@ func (s *UpdaterService) logResultItemsInternal(ctx context.Context, result *arc
case string(updater.StatusUpdated):
severity = event.EventSeveritySuccess
}
- s.recordAutoUpdateEventInternal(ctx, severity, database.JSON{
+ s.recordAutoUpdateEvent(ctx, severity, database.JSON{
"phase": item.ResourceType,
"resourceId": item.ResourceID,
"resourceName": item.ResourceName,
@@ -1347,28 +1138,17 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru
var errs []error
successfulSources := 0
- if s.deps.Docker == nil {
- errs = append(errs, common.Classify(common.ErrUnavailable, errors.New("docker service unavailable")))
+ if err := s.collectUsedImagesFromContainers(ctx, out); err != nil {
+ errs = append(errs, err)
+ slog.DebugContext(ctx, "collectUsedImages: failed collecting from containers", "error", err)
} else {
- dcli, err := s.deps.Docker.GetClient(ctx)
- if err != nil || dcli == nil {
- if err == nil {
- err = common.Classify(common.ErrUnavailable, errors.New("docker client unavailable"))
- }
- errs = append(errs, err)
- s.loggerInternal().DebugContext(ctx, "collectUsedImages: docker connection unavailable", "error", err)
- } else if collectUsedImagesFromContainersErr := s.collectUsedImagesFromContainersInternal(ctx, dcli, out); collectUsedImagesFromContainersErr != nil {
- errs = append(errs, collectUsedImagesFromContainersErr)
- s.loggerInternal().DebugContext(ctx, "collectUsedImages: failed collecting from containers", "error", collectUsedImagesFromContainersErr)
- } else {
- successfulSources++
- }
+ successfulSources++
}
if s.deps.Projects != nil {
- if err := s.collectUsedImagesFromProjectsInternal(ctx, out); err != nil {
+ if err := s.collectUsedImagesFromProjects(ctx, out); err != nil {
errs = append(errs, err)
- s.loggerInternal().DebugContext(ctx, "collectUsedImages: failed collecting from projects", "error", err)
+ slog.DebugContext(ctx, "collectUsedImages: failed collecting from projects", "error", err)
} else {
successfulSources++
}
@@ -1378,116 +1158,84 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru
return nil, errors.Join(errs...)
}
- s.loggerInternal().DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))
+ slog.DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))
return out, nil
}
-func (s *UpdaterService) collectUsedImagesFromContainersInternal(ctx context.Context, dcli *client.Client, out map[string]struct{}) error {
- if dcli == nil {
- return nil
+func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, out map[string]struct{}) error {
+ dcli, err := s.DockerClient(ctx)
+ if err != nil {
+ return err
}
-
- excludedContainers := s.buildExcludedContainerSetInternal(ctx)
- listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{All: false})
+ var excludedContainers map[string]bool
+ if s.deps.Settings != nil {
+ excludedContainers = docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))
+ }
+ listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{})
if err != nil {
return err
}
for _, summary := range listResult.Items {
if labels.IsUpdateDisabled(summary.Labels) {
- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container opted out by labels", "containerId", summary.ID)
+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container opted out by labels", "containerId", summary.ID)
continue
}
if docker.ContainerNameExcluded(summary.Names, excludedContainers) {
- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: skipping excluded container", "containerId", summary.ID, "names", summary.Names)
+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: skipping excluded container", "containerId", summary.ID, "names", summary.Names)
continue
}
imageRef := strings.TrimSpace(summary.Image)
if imageRef != "" && !refs.IsImageIDLikeReference(imageRef) {
- addNormalizedImageUpdateRefInternal(ctx, out, imageRef, "collectUsedImagesFromContainers", "containerId", summary.ID)
+ addNormalizedImageUpdateRef(ctx, out, imageRef, "collectUsedImagesFromContainers", "containerId", summary.ID)
continue
}
+ // The summary only names an image ID, so the image's tags and the configured reference stand in.
inspectResult, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dcli, summary.ID, client.ContainerInspectOptions{})
if inspectErr != nil {
- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)
+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)
continue
}
inspect := inspectResult.Container
if inspect.Config != nil && labels.IsUpdateDisabled(inspect.Config.Labels) {
- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container inspect labels opted out", "containerId", summary.ID)
- continue
- }
- for _, tag := range s.normalizedTagsForContainerInternal(ctx, dcli, inspect) {
- out[tag] = struct{}{}
- }
- }
- return nil
-}
-
-func (s *UpdaterService) collectUsedImagesFromComposeContainersInternal(ctx context.Context, composeContainers []container.Summary, activeProjectNames, out map[string]struct{}) {
- for _, summary := range composeContainers {
- projectName := docker.ComposeProjectLabel(summary.Labels)
- if projectName == "" {
- continue
- }
- if _, isActive := activeProjectNames[projectName]; !isActive {
- continue
- }
- if labels.IsUpdateDisabled(summary.Labels) {
- continue
- }
-
- imageRef := strings.TrimSpace(summary.Image)
- if imageRef == "" || refs.IsImageIDLikeReference(imageRef) {
+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect labels opted out", "containerId", summary.ID)
continue
}
- addNormalizedImageUpdateRefInternal(ctx, out, imageRef, "collectUsedImagesFromComposeContainers", "containerId", summary.ID)
- }
-}
-
-func (s *UpdaterService) normalizedTagsForContainerInternal(ctx context.Context, dcli *client.Client, inspect container.InspectResponse) []string {
- seen := map[string]struct{}{}
-
- if dcli != nil {
- if imageInspect, err := dcli.ImageInspect(ctx, inspect.Image); err == nil {
+ if imageInspect, imageErr := dcli.ImageInspect(ctx, inspect.Image); imageErr == nil {
for _, tag := range imageInspect.RepoTags {
- if strings.TrimSpace(tag) == "" || tag == "<none>:<none>" {
- continue
+ if strings.TrimSpace(tag) != "" && tag != "<none>:<none>" {
+ addNormalizedImageUpdateRef(ctx, out, tag, "normalizedTagsForContainer repo tag", "imageId", inspect.Image)
}
- addNormalizedImageUpdateRefInternal(ctx, seen, tag, "normalizedTagsForContainer repo tag", "imageId", inspect.Image)
}
}
+ if inspect.Config != nil && inspect.Config.Image != "" {
+ addNormalizedImageUpdateRef(ctx, out, inspect.Config.Image, "normalizedTagsForContainer config image", "imageId", inspect.Image)
+ }
}
-
- if inspect.Config != nil && inspect.Config.Image != "" {
- addNormalizedImageUpdateRefInternal(ctx, seen, inspect.Config.Image, "normalizedTagsForContainer config image", "imageId", inspect.Image)
- }
-
- out := slices.Collect(maps.Keys(seen))
- return out
-}
-
-func (s *UpdaterService) buildExcludedContainerSetInternal(ctx context.Context) map[string]bool {
- if s.deps.Settings == nil {
- return nil
- }
- return docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))
+ return nil
}
-func (s *UpdaterService) collectUsedImagesFromProjectsInternal(ctx context.Context, out map[string]struct{}) error {
- if s.deps.Projects == nil {
- return nil
- }
-
+func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, out map[string]struct{}) error {
allProjects, err := s.deps.Projects.ListAllProjects(ctx)
if err != nil {
return err
}
- activeProjectNames := activeComposeProjectNameSetInternal(allProjects)
+ activeProjectNames := map[string]struct{}{}
+ for _, project := range allProjects {
+ name := strings.TrimSpace(project.Name)
+ running := project.Status == projectpkg.ProjectStatusRunning || project.Status == projectpkg.ProjectStatusPartiallyRunning
+ if project.IsArchived || !running || name == "" {
+ continue
+ }
+ activeProjectNames[name] = struct{}{}
+ if normalized := projects.NormalizeProjectName(name); normalized != "" {
+ activeProjectNames[normalized] = struct{}{}
+ }
+ }
if len(activeProjectNames) == 0 {
return nil
}
@@ -1505,62 +1253,29 @@ func (s *UpdaterService) collectUsedImagesFromProjectsInternal(ctx context.Conte
return err
}
- s.collectUsedImagesFromComposeContainersInternal(ctx, composeContainers, activeProjectNames, out)
- return nil
-}
-
-func activeComposeProjectNameSetInternal(items []projectpkg.Project) map[string]struct{} {
- active := make(map[string]struct{})
- for _, project := range items {
- if project.IsArchived {
- continue
- }
- if project.Status != projectpkg.ProjectStatusRunning && project.Status != projectpkg.ProjectStatusPartiallyRunning {
+ for _, summary := range composeContainers {
+ if _, isActive := activeProjectNames[docker.ComposeProjectLabel(summary.Labels)]; !isActive || labels.IsUpdateDisabled(summary.Labels) {
continue
}
-
- name := strings.TrimSpace(project.Name)
- if name == "" {
+ imageRef := strings.TrimSpace(summary.Image)
+ if imageRef == "" || refs.IsImageIDLikeReference(imageRef) {
continue
}
- active[name] = struct{}{}
- if normalized := projects.NormalizeProjectName(name); normalized != "" {
- active[normalized] = struct{}{}
- }
+ addNormalizedImageUpdateRef(ctx, out, imageRef, "collectUsedImagesFromComposeContainers", "containerId", summary.ID)
}
- return active
+ return nil
}
-func addNormalizedImageUpdateRefInternal(ctx context.Context, out map[string]struct{}, imageRef, source string, attrs ...any) {
- normalizedRef := refs.NormalizeImageUpdateRef(imageRef)
- if normalizedRef != "" {
+func addNormalizedImageUpdateRef(ctx context.Context, out map[string]struct{}, imageRef, source string, attrs ...any) {
+ if normalizedRef := refs.NormalizeImageUpdateRef(imageRef); normalizedRef != "" {
out[normalizedRef] = struct{}{}
return
}
-
- args := slices.Clone(attrs)
- args = append(args, "source", source, "imageRef", imageRef)
- slog.DebugContext(ctx, "skipping invalid image reference", args...)
-}
-
-func (s *UpdaterService) tagFrozenPullInternal(ctx context.Context, pulledRef, imageRef string) error {
- if pulledRef == imageRef {
- return nil
- }
- dockerClient, err := s.DockerClient(ctx)
- if err != nil {
- return err
- }
- _, err = dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: pulledRef, Target: imageRef})
- return err
+ slog.DebugContext(ctx, "skipping invalid image reference", append(attrs, "source", source, "imageRef", imageRef)...)
}
-type updateAdmissionKeyInternal struct{}
-
-var errUpdateBusyInternal = errors.New("another container update is running")
-
-func (s *UpdaterService) acquireUpdateInternal(ctx context.Context) (context.Context, func(), error) {
- if s.admission == nil || ctx.Value(updateAdmissionKeyInternal{}) == s {
+func (s *UpdaterService) acquireUpdate(ctx context.Context) (context.Context, func(), error) {
+ if s.admission == nil || ctx.Value(updateAdmissionKey{}) == s {
return ctx, func() {}, nil
}
lease, admitted, err := s.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: "updater"})
@@ -1568,26 +1283,39 @@ func (s *UpdaterService) acquireUpdateInternal(ctx context.Context) (context.Con
return ctx, nil, err
}
if !admitted {
- return ctx, nil, errUpdateBusyInternal
+ return ctx, nil, errUpdateBusy
}
- return context.WithValue(ctx, updateAdmissionKeyInternal{}, s), func() { lease.Release(ctx) }, nil
+ return context.WithValue(ctx, updateAdmissionKey{}, s), func() { lease.Release(ctx) }, nil
}
-type (
- updateProgressKeyInternal struct{}
- updateProgressInternal struct {
- mu sync.Mutex
- err error
- selfTriggered bool
- selfID string
- }
-)
-
// RecordUpdateRun persists one updater resource result into Arcane history.
func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.ResourceResult) error {
var recordErr error
if s != nil && s.deps.DB != nil {
- recordErr = s.recordRunInternal(ctx, resourceResultFromModuleInternal(result))
+ now := time.Now()
+ record := &AutoUpdateRecord{
+ ResourceID: result.ResourceID,
+ ResourceType: string(result.ResourceType),
+ ResourceName: result.ResourceName,
+ Status: AutoUpdateStatus(result.Status),
+ StartTime: now,
+ EndTime: &now,
+ UpdateAvailable: result.UpdateAvailable || result.Status == updater.StatusUpdated || result.Status == updater.StatusUpdateAvailable,
+ UpdateApplied: result.UpdateApplied,
+ }
+ if result.OldImage != "" {
+ record.OldImageVersions = database.JSON{"main": result.OldImage}
+ }
+ if result.NewImage != "" {
+ record.NewImageVersions = database.JSON{"main": result.NewImage}
+ }
+ if len(result.Details) > 0 {
+ record.Details = maps.Clone(result.Details)
+ }
+ if result.Error != "" {
+ record.Error = &result.Error
+ }
+ recordErr = s.deps.DB.WithContext(ctx).Create(record).Error
}
name := cmp.Or(strings.TrimSpace(result.ResourceName), result.ResourceID)
message := name + ": " + string(result.Status)
@@ -1614,38 +1342,22 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Res
message += " (" + reason + ")"
}
}
- if activityID := activityIDFromContextInternal(ctx); activityID != "" && s != nil && s.deps.Activity != nil {
+ activityID := activityIDFromContext(ctx)
+ if activityID != "" && s != nil && s.deps.Activity != nil {
if _, appendErr := s.deps.Activity.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{Level: level, Message: message, Step: "Applying updates"}); appendErr != nil {
slog.DebugContext(ctx, "failed to append update result activity message", "activityId", activityID, "resource", name, "error", appendErr)
}
}
+ progressMessage := result.Error
var evidenceErr error
if status == scheduler.Succeeded {
- evidenceErr = s.recovery.VerifyResult(ctx, result.ResourceID)
- }
- if evidenceErr != nil {
- status = scheduler.NeedsAttention
- }
- progressMessage := result.Error
- if evidenceErr != nil {
- progressMessage = evidenceErr.Error()
+ if evidenceErr = s.recovery.VerifyResult(ctx, result.ResourceID); evidenceErr != nil {
+ status, progressMessage = scheduler.NeedsAttention, evidenceErr.Error()
+ }
}
- progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ID: result.ResourceID,
- ResourceType: string(
- result.ResourceType,
- ),
- Status: status,
- Message: progressMessage,
- ActivityID: activityIDFromContextInternal(
- ctx,
- ),
- },
- )
- err := errors.Join(recordErr, progressErr, evidenceErr)
- if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok && err != nil {
+ target := scheduler.TargetOutcome{ID: result.ResourceID, ResourceType: string(result.ResourceType), Status: status, Message: progressMessage, ActivityID: activityID}
+ err := errors.Join(recordErr, jobcontext.Progress(ctx, target), evidenceErr)
+ if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok && err != nil {
progress.mu.Lock()
progress.err = errors.Join(progress.err, err)
progress.mu.Unlock()
@@ -1653,12 +1365,12 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Res
return err
}
-func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {
- activityID := activityIDFromContextInternal(ctx)
+func (p *updateProgress) completeBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {
+ activityID := activityIDFromContext(ctx)
p.mu.Lock()
err = errors.Join(err, p.err)
recordingFailed := p.err != nil
- _, durableRun := jobcontext.Run(ctx)
+ previous, durableRun := jobcontext.Run(ctx)
selfTriggered := p.selfTriggered && durableRun
selfID := p.selfID
p.mu.Unlock()
@@ -1674,23 +1386,26 @@ func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, opti
status = scheduler.NeedsAttention
}
if selfTriggered {
- err = errors.Join(
- err,
- jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ID: selfID,
- ResourceType: "container",
- Status: scheduler.NeedsAttention,
- ActivityID: activityID,
- Message: "Self-update completion requires review",
- },
- ),
- )
+ selfTarget := scheduler.TargetOutcome{
+ ID: selfID,
+ ResourceType: "container",
+ Status: scheduler.NeedsAttention,
+ ActivityID: activityID,
+ Message: "Self-update completion requires review",
+ }
+ err = errors.Join(err, jobcontext.Progress(ctx, selfTarget), errors.New("self-update was triggered but completion requires review"))
status = scheduler.NeedsAttention
- err = errors.Join(err, errors.New("self-update was triggered but completion requires review"))
}
- batchType := updateBatchTypeInternal(ctx, options, batchCompleted)
+ // A scoped rerun of a durable run is a retry unless that run already recorded a finished batch.
+ batchType := "update-batch"
+ if durableRun && len(options.ResourceIds) > 0 && !slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == scheduler.Succeeded || target.Status == scheduler.Partial)
+ }) {
+ batchType = "update-retry"
+ }
+ if !batchCompleted {
+ batchType = "update-interrupted"
+ }
checkpoint := scheduler.TargetOutcome{ID: "auto-update", ResourceType: batchType, Status: status, ActivityID: activityID}
if err != nil {
checkpoint.Message = err.Error()
@@ -1715,7 +1430,7 @@ func (s *UpdaterService) ListTags(ctx context.Context, imageRef string) ([]strin
if s.deps.Projects != nil {
resolved, err := s.deps.Projects.ResolveRegistryCredentials(ctx)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("resolve registry credentials: %w", err)
}
credentials = resolved
}
@@ -1727,78 +1442,10 @@ func (s *UpdaterService) UpdateServiceImages(ctx context.Context, projectID stri
if s.deps.Projects == nil {
return errors.New("project service unavailable")
}
- s.appendAutoUpdateActivityMessageInternal(ctx, activityIDFromContextInternal(ctx), "Updating Compose image references", "Persisting image updates", 50)
+ s.appendAutoUpdateActivityMessage(ctx, activityIDFromContext(ctx), "Updating Compose image references", "Persisting image updates", 50)
return s.deps.Projects.UpdateProjectServiceImages(ctx, projectID, changes, s.deps.SystemUser)
}
-func (s *UpdaterService) scopedPendingRecordsInternal(ctx context.Context, records []imageupdate.ImageUpdateRecord) ([]updater.ImageUpdateRecord, error) {
- var scopedCount int64
- if err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {
- return nil, err
- }
- hasScoped := scopedCount > 0
- if !hasScoped {
- out := make([]updater.ImageUpdateRecord, 0, len(records))
- for _, record := range records {
- out = append(out, imageUpdateRecordToModuleInternal(record))
- }
- return out, nil
- }
- dockerClient, err := s.DockerClient(ctx)
- if err != nil {
- return nil, err
- }
- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
- if err != nil {
- return nil, err
- }
- out := make([]updater.ImageUpdateRecord, 0, len(records))
- for _, record := range records {
- converted := imageUpdateRecordToModuleInternal(record)
- if record.ContainerID != "" {
- out = append(out, converted)
- continue
- }
- for _, cnt := range listed.Items {
- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
- if policyErr != nil || resolved.Strategy == "tag" {
- continue
- }
- if refs.NormalizeImageUpdateRef(cnt.Image) != refs.NormalizeImageUpdateRef(converted.ImageRef()) {
- continue
- }
- converted.ContainerID = cnt.ID
- out = append(out, converted)
- }
- }
- return out, nil
-}
-
-func (s *UpdaterService) clearUnscopedRecordInternal(ctx context.Context, record updater.ImageUpdateRecord) error {
- dockerClient, err := s.DockerClient(ctx)
- if err != nil {
- return err
- }
- target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())
- if err != nil {
- return err
- }
- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
- if err != nil {
- return err
- }
- for _, cnt := range listed.Items {
- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
- if policyErr != nil || resolved.Strategy == "tag" {
- continue
- }
- if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {
- return nil
- }
- }
- return s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("id = ? AND container_id = ?", record.ID, "").Update("has_update", false).Error
-}
-
// CheckProjectUpdates checks effective Compose service policies without deploying containers.
func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID string) (*projecttypes.UpdateInfo, error) {
if s.deps.Projects == nil || s.deps.DB == nil || s.engine == nil {
@@ -1816,7 +1463,7 @@ func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID stri
if strings.TrimSpace(service.Image) == "" {
continue
}
- records = append(records, s.checkProjectServiceInternal(ctx, projectID, service))
+ records = append(records, s.checkProjectService(ctx, projectID, service))
}
if transactionErr := s.deps.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if deleteProjectUpdatesErr := tx.Where("project_id = ?", projectID).Delete(&imageupdate.ImageUpdateRecord{}).Error; deleteProjectUpdatesErr != nil {
@@ -1832,16 +1479,13 @@ func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID stri
return projectpkg.BuildConfiguredUpdateInfo(projectID, details.Services, nil, records), nil
}
-func (s *UpdaterService) checkProjectServiceInternal(ctx context.Context, projectID string, service types.ServiceConfig) imageupdate.ImageUpdateRecord {
+func (s *UpdaterService) checkProjectService(ctx context.Context, projectID string, service types.ServiceConfig) imageupdate.ImageUpdateRecord {
record := imageupdate.ImageUpdateRecord{
ID: "project::" + projectID + "::" + service.Name,
ProjectID: projectID,
ServiceName: service.Name,
- PolicyKey: imageref.UpdatePolicyKey(
- service.Image,
- service.Labels,
- ),
- CheckTime: time.Now().UTC(),
+ PolicyKey: imageref.UpdatePolicyKey(service.Image, service.Labels),
+ CheckTime: time.Now().UTC(),
}
parsed, err := refs.NormalizeReference(service.Image)
if err != nil {
@@ -1890,40 +1534,222 @@ func (s *UpdaterService) checkProjectServiceInternal(ctx context.Context, projec
return record
}
-func updateBatchTypeInternal(ctx context.Context, options arcaneupdater.Options, batchCompleted bool) string {
- batchType := "update-batch"
- if previous, ok := jobcontext.Run(ctx); ok && len(options.ResourceIds) > 0 {
- batchType = "update-retry"
- for _, target := range previous.Outcome.Targets {
- if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == scheduler.Succeeded || target.Status == scheduler.Partial) {
- batchType = "update-batch"
- }
- }
+// RegisterWorkflows defines the auto-update workflow (fresh candidate check, frozen plan without
+// failed checks, confirm-first apply) and the container-update workflow. Call it before the host starts.
+func (s *UpdaterService) RegisterWorkflows(engine *flow.Engine) error {
+ autoUpdate, err := engine.Define(flow.Definition{
+ Name: "auto-update",
+ Version: 5,
+ Fingerprint: "91e27dc48df6737110ec334dfa9c2e2e032a6cc4285e8f1dd08dc40741e6d5b9",
+ Concurrency: 1,
+ Timeout: 2 * time.Hour,
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeAutoUpdate,
+ Queue: true,
+ ResourceType: new("system"),
+ ResourceName: new("Auto update"),
+ Step: "Planning updates",
+ LatestMessage: "Auto-update run started",
+ Metadata: database.JSON{"dryRun": false},
+ },
+ Labels: map[string]string{
+ "resume": "Checking for a saved plan",
+ "candidates": "Selecting update candidates",
+ "check": "Checking images",
+ "plan": "Planning updates",
+ "apply": "Applying updates",
+ "finalize": "Finishing auto-update",
+ },
+ Steps: []workflow.StepSpec{
+ workflow.Step("resume", engine.Handler(s.autoUpdateResume)),
+ // A retry keeps its frozen plan, so it skips selecting and checking images.
+ workflow.Step("candidates", engine.Handler(s.autoUpdateCandidates), workflow.WithSkipIf("resume", true)),
+ workflow.Child("check", workflow.WithDefinition(s.deps.ImageUpdates.CheckWorkflow().Francis()), workflow.WithSkipIf("resume", true)),
+ workflow.Step("plan", engine.Handler(s.planAutoUpdate)),
+ workflow.Step("apply", engine.Handler(s.applyAutoUpdate),
+ workflow.WithMaxAttempts(5), workflow.WithRetryBackoff(30*time.Second, 5*time.Minute)),
+ workflow.Step("finalize", engine.Handler(s.finalizeAutoUpdate)),
+ },
+ })
+ if err != nil {
+ return err
}
- if !batchCompleted {
- batchType = "update-interrupted"
+ s.autoUpdate = autoUpdate
+ return s.execution.RegisterWorkflows(engine)
+}
+
+// AutoUpdateWorkflow is the durable auto-update workflow.
+func (s *UpdaterService) AutoUpdateWorkflow() *flow.Workflow { return s.autoUpdate }
+
+// autoUpdateCandidates selects the monitored images of auto-update eligible containers.
+func (s *UpdaterService) autoUpdateCandidates(ctx context.Context, t flow.Task) (any, error) {
+ used, err := s.CollectUsedImages(ctx)
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ monitored, err := s.deps.ImageUpdates.MonitoredImageRefs(ctx, slices.Sorted(maps.Keys(used)))
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
}
- return batchType
+ return t.ChildInput(imageupdatetypes.CheckRequest{ImageRefs: monitored})
}
-// RegisterActors registers durable single-container update delivery.
-func (s *UpdaterService) RegisterActors(runtime *francis.Runtime) error {
- return s.execution.RegisterActors(runtime)
+// autoUpdateResume reports whether the run already holds a frozen plan.
+func (s *UpdaterService) autoUpdateResume(ctx context.Context, _ flow.Task) (any, error) {
+ run, ok := jobcontext.Run(ctx)
+ return ok && s.recovery.Planned(run), nil
}
-// Start repairs persisted single-container delivery intents until shutdown.
-func (s *UpdaterService) Start(ctx context.Context) error {
- return s.execution.Start(ctx)
+// planAutoUpdate freezes the update plan without the images and containers whose check failed.
+func (s *UpdaterService) planAutoUpdate(ctx context.Context, t flow.Task) (any, error) {
+ run, ok := jobcontext.Run(ctx)
+ if !ok {
+ return nil, errors.New("auto-update requires a job run")
+ }
+ var checked scheduler.Outcome
+ if err := t.DecodeOutput("check", &checked); err != nil && !errors.Is(err, workflow.ErrStepSkipped) {
+ return nil, err
+ }
+ excluded := map[string]bool{}
+ for _, target := range checked.Targets {
+ // Failed checks are run evidence, so a retry or recovery that skips the check still reports them.
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ switch target.ResourceType {
+ case "image":
+ excluded[refs.NormalizeImageUpdateRef(target.ID)] = true
+ case "container":
+ excluded[target.ID] = true
+ }
+ }
+ if _, err := s.recovery.FreezePending(contextWithActivityID(ctx, t.ActivityID()), run, excluded); err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ return nil, nil
}
-// Stop cancels business execution before the actor host drains jobs.
-func (s *UpdaterService) Stop(ctx context.Context) error {
- return s.execution.Stop(ctx)
+// applyAutoUpdate confirms every frozen target on each delivery before
+// applying the ones still unchanged, so redelivery never repeats a Docker change.
+func (s *UpdaterService) applyAutoUpdate(ctx context.Context, t flow.Task) (any, error) {
+ run, ok := jobcontext.Run(ctx)
+ if !ok {
+ return nil, errors.New("auto-update requires a job run")
+ }
+ ctx, release, err := s.acquireUpdate(ctx)
+ if errors.Is(err, errUpdateBusy) {
+ // A plan whose every target already settled needs no updater, so it finishes normally.
+ if unsettled, unsettledErr := s.recovery.Unsettled(run); unsettledErr == nil && unsettled == 0 {
+ return autoUpdateApplied{Completed: true}, nil
+ }
+ // Waiting would hold this run's activity slot against the update holding the updater, so the run ends now:
+ // a plan that already started applying stays retryable, and an untouched one skips as the job always has.
+ return autoUpdateApplied{Busy: true, Started: s.recovery.Started(run)}, nil
+ }
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ defer release()
+ start := time.Now()
+ ctx = contextWithActivityID(ctx, t.ActivityID())
+ progress := &updateProgress{}
+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)
+
+ ctx, remaining, unresolved, err := s.recovery.ResumePlan(ctx, run)
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ applied := autoUpdateApplied{Unresolved: unresolved, Completed: true}
+ if remaining > 0 {
+ if applyErr := s.applyBatch(ctx, arcaneupdater.Options{}, &applied.Result); applyErr != nil {
+ // A shutdown hands the task back so the redelivery finishes the remaining frozen targets.
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ applied.Error, applied.Completed = applyErr.Error(), false
+ }
+ }
+ applied.Result.Items = nil
+ applied.Result.Duration = time.Since(start).String()
+ progress.mu.Lock()
+ if progress.err != nil {
+ applied.RecordingError = progress.err.Error()
+ }
+ applied.SelfTriggered, applied.SelfID = progress.selfTriggered, progress.selfID
+ progress.mu.Unlock()
+ return applied, nil
}
-// ActiveUpdateActivityIDs lists activities owned by persisted single-container updates.
-func (s *UpdaterService) ActiveUpdateActivityIDs(ctx context.Context) ([]string, error) {
- return s.execution.ActiveUpdateActivityIDs(ctx)
+// finalizeAutoUpdate records the batch checkpoint and returns the run outcome.
+func (s *UpdaterService) finalizeAutoUpdate(ctx context.Context, t flow.Task) (any, error) {
+ var applied autoUpdateApplied
+ if err := t.DecodeOutput("apply", &applied); err != nil {
+ return nil, err
+ }
+ activityID := t.ActivityID()
+ ctx = contextWithActivityID(ctx, activityID)
+ if applied.Busy {
+ status, message := scheduler.Skipped, "Another update is active"
+ if applied.Started {
+ status, message = scheduler.Failed, "Another update is active; the remaining targets were not updated"
+ }
+ batch := scheduler.TargetOutcome{ID: "auto-update", ResourceType: "update-batch", Status: status, ActivityID: activityID, Message: message}
+ if err := jobcontext.Progress(ctx, batch); err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ return scheduler.Outcome{Status: status, Message: message, ActivityID: activityID}, nil
+ }
+ progress := &updateProgress{selfTriggered: applied.SelfTriggered, selfID: applied.SelfID}
+ if applied.RecordingError != "" {
+ progress.err = errors.New(applied.RecordingError)
+ }
+ var applyErr error
+ if applied.Error != "" {
+ applyErr = errors.New(applied.Error)
+ }
+ result := &applied.Result
+ // Containers that could not be planned never reached apply, so they count as failed here.
+ run, _ := jobcontext.Run(ctx)
+ unplanned := s.recovery.Unplanned(run)
+ result.Failed += len(unplanned)
+ err := progress.completeBatch(ctx, arcaneupdater.Options{}, result, applied.Completed, applyErr)
+ slog.InfoContext(ctx, "auto-update run completed",
+ "checked", result.Checked,
+ "updated", result.Updated,
+ "restarted", result.Restarted,
+ "skipped", result.Skipped,
+ "failed", result.Failed,
+ )
+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Auto-update run completed", ActivityID: activityID}
+ if result.Failed > 0 || err != nil {
+ outcome.Status = scheduler.Partial
+ outcome.Message = "Some updates failed"
+ if err != nil {
+ outcome.Message += ": " + err.Error()
+ }
+ }
+ outcome.Targets = append(outcome.Targets, unplanned...)
+ if applied.Unresolved {
+ outcome.Status = scheduler.Failed
+ outcome.Message = "Some target outcomes could not be confirmed"
+ }
+ if outcomeErr, ok := errors.AsType[*scheduler.OutcomeError](err); ok {
+ outcome.Status = outcomeErr.Outcome.Status
+ outcome.Message = outcomeErr.Outcome.Message
+ }
+ return outcome, nil
+}
+
+// ValidateAutoUpdateRetry allows a retry only while frozen targets remain unsettled.
+func (s *UpdaterService) ValidateAutoUpdateRetry(_ context.Context, run scheduler.Run) error {
+ unsettled, err := s.recovery.Unsettled(run)
+ if err != nil {
+ return fmt.Errorf("auto-update has no frozen plan to retry: %w", err)
+ }
+ if unsettled == 0 {
+ return errors.New("auto-update has no unsettled targets to retry")
+ }
+ return nil
}
// ReconcilePending resumes only frozen targets whose original container is unchanged.
diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.go
index 41483489c..51c11f429 100644
--- a/backend/internal/updater/service_test.go
+++ b/backend/internal/updater/service_test.go
@@ -32,7 +32,7 @@ import (
"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"go.getarcane.app/updater"
"go.getarcane.app/updater/labels"
@@ -53,21 +53,20 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/getarcaneapp/arcane/backend/v2/internal/updater/children/execution"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
)
-type fakeDockerClientProviderInternal struct {
+type fakeDockerClientProvider struct {
client any
}
-func (f fakeDockerClientProviderInternal) DockerClient(context.Context) (*client.Client, error) {
+func (f fakeDockerClientProvider) DockerClient(context.Context) (*client.Client, error) {
cli, _ := f.client.(*client.Client)
if cli == nil {
return nil, errors.New("docker client unavailable")
@@ -75,12 +74,12 @@ func (f fakeDockerClientProviderInternal) DockerClient(context.Context) (*client
return cli, nil
}
-type fakeImagePullerInternal struct {
+type fakeImagePuller struct {
pulled []string
fail map[string]error
}
-func (f *fakeImagePullerInternal) PullImage(_ context.Context, imageRef string, _ io.Writer) error {
+func (f *fakeImagePuller) PullImage(_ context.Context, imageRef string, _ io.Writer) error {
f.pulled = append(f.pulled, imageRef)
if f.fail == nil {
return nil
@@ -88,44 +87,44 @@ func (f *fakeImagePullerInternal) PullImage(_ context.Context, imageRef string,
return f.fail[imageRef]
}
-type fakeRunRecorderInternal struct {
+type fakeRunRecorder struct {
results []updater.ResourceResult
}
-func (f *fakeRunRecorderInternal) RecordUpdateRun(_ context.Context, result updater.ResourceResult) error {
+func (f *fakeRunRecorder) RecordUpdateRun(_ context.Context, result updater.ResourceResult) error {
f.results = append(f.results, result)
return nil
}
-type fakeSettingsProviderInternal struct{}
+type fakeSettingsProvider struct{}
-func (fakeSettingsProviderInternal) ExcludedContainers(context.Context) ([]string, error) {
+func (fakeSettingsProvider) ExcludedContainers(context.Context) ([]string, error) {
return nil, nil
}
-type fakeUsedImageCollectorInternal struct {
+type fakeUsedImageCollector struct {
images map[string]struct{}
}
-func (f fakeUsedImageCollectorInternal) UsedImages(context.Context) (map[string]struct{}, error) {
+func (f fakeUsedImageCollector) UsedImages(context.Context) (map[string]struct{}, error) {
return f.images, nil
}
-type fakeProjectUpdaterInternal struct {
+type fakeProjectUpdater struct {
projects map[string]updater.ComposeProject
updateErrs map[string]error
calls []string
imageChanges map[string]map[string]updatertypes.ServiceImageChange
}
-func (f *fakeProjectUpdaterInternal) ProjectByComposeName(_ context.Context, composeName string) (updater.ComposeProject, error) {
+func (f *fakeProjectUpdater) ProjectByComposeName(_ context.Context, composeName string) (updater.ComposeProject, error) {
if localProject, ok := f.projects[composeName]; ok {
return localProject, nil
}
return updater.ComposeProject{}, errors.New("project not found")
}
-func (f *fakeProjectUpdaterInternal) UpdateServices(_ context.Context, projectID string, services []string) error {
+func (f *fakeProjectUpdater) UpdateServices(_ context.Context, projectID string, services []string) error {
f.calls = append(f.calls, projectID+":"+strings.Join(services, ","))
if f.updateErrs == nil {
return nil
@@ -133,7 +132,7 @@ func (f *fakeProjectUpdaterInternal) UpdateServices(_ context.Context, projectID
return f.updateErrs[projectID]
}
-func (f *fakeProjectUpdaterInternal) UpdateServiceImages(_ context.Context, projectID string, changes map[string]updatertypes.ServiceImageChange) error {
+func (f *fakeProjectUpdater) UpdateServiceImages(_ context.Context, projectID string, changes map[string]updatertypes.ServiceImageChange) error {
if f.imageChanges == nil {
f.imageChanges = map[string]map[string]updatertypes.ServiceImageChange{}
}
@@ -143,7 +142,7 @@ func (f *fakeProjectUpdaterInternal) UpdateServiceImages(_ context.Context, proj
return f.updateErrs[projectID]
}
-func newUpdaterApplyPendingDockerServerInternal(
+func newUpdaterApplyPendingDockerServer(
t *testing.T,
containers []container.Summary,
verificationByService map[string][]container.Summary,
@@ -224,32 +223,31 @@ func newUpdaterApplyPendingDockerServerInternal(
return server
}
-type mockSystemUpgradeServiceInternal struct {
+type mockSystemUpgradeService struct {
triggerCalled bool
triggerError error
capturedUser *usertypes.Actor
capturedTarget *updater.SelfUpdateTarget
}
-func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {
+func (m *mockSystemUpgradeService) TriggerUpgradeViaCLI(_ context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {
m.triggerCalled = true
m.capturedUser = &user
m.capturedTarget = &target
return "upgrader-container-id", m.triggerError
}
-func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {
+func TestUpdaterService_ApplyPendingNoRecords(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var checkpoints []scheduler.TargetOutcome
ctx = jobcontext.WithExecution(ctx, scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error { checkpoints = append(checkpoints, target); return nil })
result, err := svc.ApplyPending(ctx, arcaneupdater.Options{DryRun: true})
- require.Len(t, checkpoints, 2)
- assert.Equal(t, scheduler.Running, checkpoints[0].Status)
- assert.Equal(t, scheduler.Succeeded, checkpoints[1].Status)
+ require.Len(t, checkpoints, 1)
+ assert.Equal(t, scheduler.Succeeded, checkpoints[0].Status)
require.NoError(t, err)
require.NotNil(t, result)
@@ -265,41 +263,18 @@ func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {
require.ErrorAs(t, err, &outcomeErr)
assert.Equal(t, scheduler.NeedsAttention, outcomeErr.Outcome.Status)
checkpoints = nil
- ctx = jobcontext.WithExecution(
- t.Context(),
- scheduler.Run{AttemptCount: 1},
- func(target scheduler.TargetOutcome) error {
- checkpoints = append(
- checkpoints,
- target,
- )
- return nil
- },
- )
+ ctx = jobcontext.WithExecution(t.Context(), scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error {
+ checkpoints = append(checkpoints, target)
+ return nil
+ })
svc.engine = nil
require.Panics(t, func() { _, _ = svc.ApplyPending(ctx, arcaneupdater.Options{}) })
require.NotEmpty(t, checkpoints)
assert.Equal(t, scheduler.NeedsAttention, checkpoints[len(checkpoints)-1].Status)
}
-// Type and ResourceIds are Arcane-side scoping the engine never acted on;
-// ApplyPending handles a scoped request itself before reaching the engine.
-// A pending run, forced or scoped, never lifts settings exclusions: that
-// override belongs to UpdateSingleContainer alone.
-func TestUpdaterService_ModuleOptionsFromUpdaterOptionsInternal(t *testing.T) {
- got := moduleOptionsFromUpdaterOptionsInternal(arcaneupdater.Options{
- Type: string(updater.ResourceTypeContainer),
- ResourceIds: []string{"container-1", "container-2"},
- ForceUpdate: true,
- DryRun: true,
- })
-
- assert.Equal(t, updater.Options{Force: true, DryRun: true}, got)
- assert.False(t, got.IgnoreSettingsExclusions)
-}
-
-func TestUpdaterService_ResultFromModulePreservesRestartedInternal(t *testing.T) {
- got := resultFromModuleInternal(&updater.Result{
+func TestUpdaterService_ResultFromModulePreservesRestarted(t *testing.T) {
+ got := resultFromModule(&updater.Result{
Success: true,
Checked: 2,
Updated: 1,
@@ -322,11 +297,11 @@ func TestUpdaterService_ResultFromModulePreservesRestartedInternal(t *testing.T)
assert.True(t, got.Items[0].UpdateApplied)
}
-func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
+func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
ctx := t.Context()
t.Run("server label triggers upgrade with system user", func(t *testing.T) {
- mockUpgrade := &mockSystemUpgradeServiceInternal{}
+ mockUpgrade := &mockSystemUpgradeService{}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -347,7 +322,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {
- mockUpgrade := &mockSystemUpgradeServiceInternal{}
+ mockUpgrade := &mockSystemUpgradeService{}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -370,7 +345,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
t.Run("agent label triggers upgrade", func(t *testing.T) {
- mockUpgrade := &mockSystemUpgradeServiceInternal{}
+ mockUpgrade := &mockSystemUpgradeService{}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -383,7 +358,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {
- mockUpgrade := &mockSystemUpgradeServiceInternal{}
+ mockUpgrade := &mockSystemUpgradeService{}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -409,7 +384,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
t.Run("upgrade errors are wrapped", func(t *testing.T) {
- mockUpgrade := &mockSystemUpgradeServiceInternal{triggerError: errors.New("upgrade failed")}
+ mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -422,7 +397,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
}
-func TestUpdaterService_StatusTrackingInternal(t *testing.T) {
+func TestUpdaterService_StatusTracking(t *testing.T) {
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -445,13 +420,13 @@ func TestUpdaterService_StatusTrackingInternal(t *testing.T) {
assert.Empty(t, status.ProjectIds)
}
-func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
+func TestUpdaterService_DockerClientAdapter(t *testing.T) {
ctx := t.Context()
t.Run("active engine updates correlate before the stop event is recorded", func(t *testing.T) {
events := event.NewEventService(nil, nil, nil)
svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
- end := svc.engineInternal().BeginContainerUpdate("updating-container")
+ end := svc.engine.BeginContainerUpdate("updating-container")
require.True(t, events.ShouldSuppressDaemonEvent("container", "updating-container", "", ""))
require.False(t, events.ShouldSuppressDaemonEvent("container", "unrelated-container", "", ""))
require.False(t, events.ShouldSuppressDaemonEvent("image", "updating-container", "", ""))
@@ -471,8 +446,8 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
})
t.Run("delegates to configured docker service", func(t *testing.T) {
- server := newImagePullServerWithObserverInternal(t, nil, nil)
- wantClient := newTestDockerClientInternal(t, server)
+ server := newImagePullServerWithObserver(t, nil, nil)
+ wantClient := newTestDockerClient(t, server)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(wantClient)
svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -484,7 +459,7 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
})
}
-func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
+func TestUpdaterService_PullImageAdapter(t *testing.T) {
ctx := t.Context()
t.Run("missing image service returns unavailable error", func(t *testing.T) {
@@ -498,9 +473,9 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
})
t.Run("delegates to Arcane image puller", func(t *testing.T) {
- db := setupProjectTestDBInternal(t)
- server := newImagePullServerWithObserverInternal(t, nil, nil)
- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ db := setupProjectTestDB(t)
+ server := newImagePullServerWithObserver(t, nil, nil)
+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
imageSvc := image.NewImageService(db, dockerSvc, nil, nil, nil, event.NewEventService(db, nil, nil), nil, nil)
svc, svcErr := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -513,22 +488,22 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
})
t.Run("passes database registry credentials to Arcane image puller", func(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&registry.ContainerRegistry{}, &kv.KVEntry{}))
crypto.InitEncryption(&crypto.Config{
Environment: string(config.AppEnvironmentTest),
EncryptionKey: "test-encryption-key-for-testing-32bytes-min",
})
- createTestPullRegistryInternal(t, db, "https://registry.example.com", "db-user", "db-token")
+ createTestPullRegistry(t, db, "https://registry.example.com", "db-user", "db-token")
imageRef := "registry.example.com/team/app:latest"
var gotAuth string
- server := newImagePullServerWithObserverInternal(t, nil, func(fullRef, authHeader string) {
+ server := newImagePullServerWithObserver(t, nil, func(fullRef, authHeader string) {
if fullRef == imageRef {
gotAuth = authHeader
}
})
- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
imageSvc := image.NewImageService(db, dockerSvc, registrySvc, nil, nil, event.NewEventService(db, nil, nil), nil, nil)
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)
@@ -542,16 +517,16 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, gotAuth)
- authConfig := decodeRegistryAuthInternal(t, gotAuth)
+ authConfig := decodeRegistryAuth(t, gotAuth)
assert.Equal(t, "db-user", authConfig.Username)
assert.Equal(t, "db-token", authConfig.Password)
assert.Contains(t, progress.String(), "Pulled")
})
}
-func TestUpdaterService_PendingImageUpdatesAdapterInternal(t *testing.T) {
+func TestUpdaterService_PendingImageUpdatesAdapter(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
latest := "1.2.4"
currentDigest := "sha256:1111111111111111111111111111111111111111111111111111111111111111"
latestDigest := "sha256:2222222222222222222222222222222222222222222222222222222222222222"
@@ -597,10 +572,10 @@ func TestUpdaterService_PendingImageUpdatesAdapterInternal(t *testing.T) {
assert.Equal(t, &lastError, records[0].LastError)
}
-// TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal guards
-// newEmptyContainerListDockerServiceInternal stubs the container listing the
+// TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications guards
+// newEmptyContainerListDockerService stubs the container listing the
// notification flush uses to resolve update-check eligibility.
-func newEmptyContainerListDockerServiceInternal(t *testing.T) *docker.DockerClientService {
+func newEmptyContainerListDockerService(t *testing.T) *docker.DockerClientService {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@@ -613,14 +588,14 @@ func newEmptyContainerListDockerServiceInternal(t *testing.T) *docker.DockerClie
http.NotFound(w, r)
}))
t.Cleanup(server.Close)
- return &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
+ return &docker.DockerClientService{Client: newTestDockerClient(t, server)}
}
// PendingImageUpdates must not hand records to the engine when the flush could
// not determine what to send (#3532): consuming them would lose the pending
// notification for good instead of retrying once Docker is reachable again.
-func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnresolvedInternal(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnresolved(t *testing.T) {
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, &notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)
@@ -640,9 +615,9 @@ func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnre
// issue #3132: the updater consumes and clears has_update records without checking
// notification_sent, so an "Updates Available" notification pending at consumption
// time was silently lost. PendingImageUpdates must flush it before the engine runs.
-func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t *testing.T) {
+func TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, &notification.NotificationSettings{}))
var calls atomic.Int32
@@ -662,7 +637,7 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t
}).Error)
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)
+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)
svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -687,13 +662,13 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t
// With no providers configured, PendingImageUpdates still returns the records and
// leaves notification_sent false (preserves the #3079 "don't mark when nothing
// delivered" semantics).
-func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotifiedInternal(t *testing.T) {
+func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotified(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, &notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)
+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)
svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -714,9 +689,9 @@ func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotifiedInternal(t
assert.False(t, reloaded.NotificationSent)
}
-func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {
+func TestUpdaterService_RecordUpdateRunAdapter(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}))
svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -744,8 +719,8 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {
assert.Equal(t, "nginx:1.2.3", record.OldImageVersions["main"])
assert.Equal(t, "nginx:1.2.4", record.NewImageVersions["main"])
assert.Equal(t, "test", record.Details["source"])
- progress := &updateProgressInternal{}
- ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)
+ progress := &updateProgress{}
+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)
ctx = jobcontext.WithExecution(ctx, scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error {
assert.Equal(t, scheduler.Failed, target.Status)
return errors.New("progress unavailable")
@@ -755,9 +730,9 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {
require.ErrorContains(t, progress.err, "progress unavailable")
}
-func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing.T) {
+func TestUpdaterService_RecordUpdateRunAppendsActivityMessage(t *testing.T) {
ctx := t.Context()
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}, &activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
@@ -816,7 +791,7 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing
started, err := activityService.StartActivity(ctx, activitylib.StartRequest{EnvironmentID: "0", Type: activitytypes.TypeAutoUpdate})
require.NoError(t, err)
- require.NoError(t, svc.RecordUpdateRun(contextWithActivityIDInternal(ctx, started.ID), tt.result))
+ require.NoError(t, svc.RecordUpdateRun(contextWithActivityID(ctx, started.ID), tt.result))
detail, err := activityService.GetActivityDetail(ctx, "0", started.ID, 10)
require.NoError(t, err)
@@ -827,26 +802,25 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing
}
}
-func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t *testing.T) {
+func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testing.T) {
for _, cancelActivity := range []bool{false, true} {
name := kit.Ternary(cancelActivity, "activity cancellation while queued", "request disconnect")
t.Run(name, func(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
sqlDB, err := db.DB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
activityService := activity.NewActivityService(db, nil)
- runtime := francistest.New(t)
- admission := runs.NewAdmission(runtime.Service(), t.Name())
- require.NoError(t, admission.Register(runtime))
+ harness := flowtest.New(t, activityService)
+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(harness.Runtime))
svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, admission, nil)
require.NoError(t, err)
- require.NoError(t, svc.RegisterActors(runtime))
- francistest.Start(t, runtime)
+ require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
- // Holding the update admission makes the executor decline the
- // dispatched job until the lease is released.
+ // Holding the update admission makes the workflow hand its task back until the lease is released.
held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater"})
require.NoError(t, err)
require.True(t, admitted)
@@ -856,14 +830,14 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t
requestCtx, cancelRequest := context.WithCancel(t.Context())
workCtx := utils.ActivityRuntimeContext(requestCtx, t.Context())
- type acceptedInternal struct {
+ type acceptResult struct {
activity *activitytypes.Activity
err error
}
- accepted := make(chan acceptedInternal, 1)
+ accepted := make(chan acceptResult, 1)
go func() {
item, acceptErr := svc.AcceptSingleContainerUpdate(workCtx, "missing-container")
- accepted <- acceptedInternal{activity: item, err: acceptErr}
+ accepted <- acceptResult{activity: item, err: acceptErr}
}()
var item *activitytypes.Activity
select {
@@ -890,31 +864,30 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t
}
}
-func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizesInternal(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizes(t *testing.T) {
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, err)
- runtime := francistest.New(t)
- require.NoError(t, svc.RegisterActors(runtime))
- // A service whose host has not started rejects durable dispatch.
+ // A host that has not started cannot start the workflow.
+ require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activityService).Engine))
accepted, err := svc.AcceptSingleContainerUpdate(t.Context(), "container-1")
require.Nil(t, accepted)
- require.ErrorContains(t, err, "persist container update")
+ require.ErrorContains(t, err, "start container-update")
var saved activity.Activity
require.NoError(t, db.First(&saved, "resource_id = ?", "container-1").Error)
require.Equal(t, activitytypes.StatusFailed, saved.Status)
- unmigrated := setupProjectTestDBInternal(t)
- svc.deps.Activity = activity.NewActivityService(unmigrated, nil)
+ unmigrated := setupProjectTestDB(t)
+ require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activity.NewActivityService(unmigrated, nil)).Engine))
accepted, err = svc.AcceptSingleContainerUpdate(t.Context(), "container-2")
require.Nil(t, accepted)
- require.ErrorContains(t, err, "start container update activity")
+ require.ErrorContains(t, err, "start container-update activity")
}
-func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing.T) {
+func TestUpdaterService_SingleContainerActivityResultMessages(t *testing.T) {
for _, tt := range []struct {
name string
result *arcaneupdater.Result
@@ -980,7 +953,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing
},
} {
t.Run(tt.name, func(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
@@ -994,7 +967,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing
},
)
require.NoError(t, err)
- svc.finishSingleContainerUpdateInternal(t.Context(), started.ID, tt.result, nil)
+ svc.finishSingleContainerUpdate(t.Context(), started.ID, tt.result, nil)
var saved activity.Activity
require.NoError(t, db.First(&saved, "id = ?", started.ID).Error)
require.Equal(t, tt.wantStatus, saved.Status)
@@ -1004,7 +977,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing
}
}
-func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInternal(t *testing.T) {
+func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjects(t *testing.T) {
ctx := t.Context()
oldRefFailed := "registry.example.com/team/fail:1.0.0"
@@ -1098,11 +1071,11 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte
},
}
- server := newUpdaterApplyPendingDockerServerInternal(t, containers, verificationByService, inspectByID, imageInspectByRef)
- dockerProvider := fakeDockerClientProviderInternal{client: newTestDockerClientInternal(t, server)}
- puller := &fakeImagePullerInternal{}
- recorder := &fakeRunRecorderInternal{}
- projectUpdater := &fakeProjectUpdaterInternal{
+ server := newUpdaterApplyPendingDockerServer(t, containers, verificationByService, inspectByID, imageInspectByRef)
+ dockerProvider := fakeDockerClientProvider{client: newTestDockerClient(t, server)}
+ puller := &fakeImagePuller{}
+ recorder := &fakeRunRecorder{}
+ projectUpdater := &fakeProjectUpdater{
projects: map[string]updater.ComposeProject{
"proj-fail": {ID: "project-fail", Name: "proj-fail"},
"proj-success": {ID: "project-success", Name: "proj-success"},
@@ -1138,9 +1111,9 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte
},
),
RunRecorder: recorder,
- Settings: fakeSettingsProviderInternal{},
+ Settings: fakeSettingsProvider{},
ProjectUpdater: projectUpdater,
- UsedImageCollector: fakeUsedImageCollectorInternal{images: map[string]struct{}{
+ UsedImageCollector: fakeUsedImageCollector{images: map[string]struct{}{
oldRefFailed: {},
oldRefUpdated: {},
}},
@@ -1181,7 +1154,7 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte
assert.Contains(t, recordedStatuses, updater.StatusUpdated)
}
-func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeInternal(t *testing.T) {
+func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(t *testing.T) {
ctx := t.Context()
oldRef := "ghcr.io/getarcaneapp/arcane:1.0.0"
@@ -1235,15 +1208,15 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI
},
}
- server := newUpdaterApplyPendingDockerServerInternal(t, containers, verificationByService, inspectByID, imageInspectByRef)
- dockerProvider := fakeDockerClientProviderInternal{client: newTestDockerClientInternal(t, server)}
- puller := &fakeImagePullerInternal{}
- projectUpdater := &fakeProjectUpdaterInternal{
+ server := newUpdaterApplyPendingDockerServer(t, containers, verificationByService, inspectByID, imageInspectByRef)
+ dockerProvider := fakeDockerClientProvider{client: newTestDockerClient(t, server)}
+ puller := &fakeImagePuller{}
+ projectUpdater := &fakeProjectUpdater{
projects: map[string]updater.ComposeProject{
"arcane": {ID: "project-arcane", Name: "arcane"},
},
}
- mockUpgrade := &mockSystemUpgradeServiceInternal{}
+ mockUpgrade := &mockSystemUpgradeService{}
svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
@@ -1259,10 +1232,10 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI
CurrentVersion: "1.0.0",
LatestVersion: new("1.0.1"),
}),
- Settings: fakeSettingsProviderInternal{},
+ Settings: fakeSettingsProvider{},
ProjectUpdater: projectUpdater,
SelfUpdater: svc,
- UsedImageCollector: fakeUsedImageCollectorInternal{images: map[string]struct{}{
+ UsedImageCollector: fakeUsedImageCollector{images: map[string]struct{}{
oldRef: {},
}},
LabelPolicy: updater.DefaultLabelPolicy(),
@@ -1291,7 +1264,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI
// The engine picks the reference Arcane's updates are applied to, so its choice
// is worth pinning here as well as in the library.
-func TestPullableImageRefInternal(t *testing.T) {
+func TestPullableImageRef(t *testing.T) {
tests := []struct {
name string
summaryImage string
@@ -1333,8 +1306,8 @@ func TestPullableImageRefInternal(t *testing.T) {
}
}
-// createTestPullRegistryInternal inserts an enabled generic registry with an encrypted token.
-func createTestPullRegistryInternal(t *testing.T, db *database.DB, localUrl, username, token string) {
+// createTestPullRegistry inserts an enabled generic registry with an encrypted token.
+func createTestPullRegistry(t *testing.T, db *database.DB, localUrl, username, token string) {
t.Helper()
encryptedToken, err := crypto.Encrypt(token)
@@ -1350,8 +1323,8 @@ func createTestPullRegistryInternal(t *testing.T, db *database.DB, localUrl, use
require.NoError(t, db.WithContext(t.Context()).Create(reg).Error)
}
-// decodeRegistryAuthInternal decodes a base64 X-Registry-Auth header.
-func decodeRegistryAuthInternal(t *testing.T, encoded string) dockerregistry.AuthConfig {
+// decodeRegistryAuth decodes a base64 X-Registry-Auth header.
+func decodeRegistryAuth(t *testing.T, encoded string) dockerregistry.AuthConfig {
t.Helper()
cfg, err := authconfig.Decode(encoded)
@@ -1359,8 +1332,8 @@ func decodeRegistryAuthInternal(t *testing.T, encoded string) dockerregistry.Aut
return *cfg
}
-// newTestDockerClientInternal builds a Docker client pointed at a fake Docker HTTP server.
-func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+// newTestDockerClient builds a Docker client pointed at a fake Docker HTTP server.
+func newTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
httpClient := server.Client()
@@ -1378,8 +1351,8 @@ func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.
return cli
}
-// newImagePullServerWithObserverInternal is NewImagePullServer with a pull callback.
-func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[string]dockertypesimage.InspectResponse, onPull func(fullRef, authHeader string)) *httptest.Server {
+// newImagePullServerWithObserver is NewImagePullServer with a pull callback.
+func newImagePullServerWithObserver(t *testing.T, inspectByRef map[string]dockertypesimage.InspectResponse, onPull func(fullRef, authHeader string)) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -1434,8 +1407,8 @@ func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[strin
return server
}
-// setupProjectTestDBInternal builds an in-memory DB migrated for project-related tests.
-func setupProjectTestDBInternal(t *testing.T) *database.DB {
+// setupProjectTestDB builds an in-memory DB migrated for project-related tests.
+func setupProjectTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -1443,8 +1416,8 @@ func setupProjectTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+func TestUpdaterService_ScopedPendingRecordsIsolation(t *testing.T) {
+ db := setupProjectTestDB(t)
imageRef := "registry.example.com/team/app:1.2.3"
stored := []imageupdate.ImageUpdateRecord{
{ID: "shared-image", Repository: "registry.example.com/team/app", Tag: "1.2.3", HasUpdate: true, UpdateType: imageupdate.UpdateTypeDigest},
@@ -1476,8 +1449,8 @@ func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {
{ID: "digest-a", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: "shared-image", State: container.StateRunning},
{ID: "digest-b", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: "shared-image", State: container.StateRunning},
}
- server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, nil)
- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, nil)
+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
records, err := svc.PendingImageUpdates(t.Context())
@@ -1503,7 +1476,7 @@ func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {
assert.ElementsMatch(t, []string{"shared-image", "container::tag-b"}, pendingIDs)
}
-func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {
+func TestUpdaterService_SharedDigestRecordClearing(t *testing.T) {
for _, tt := range []struct {
name string
remainingImage string
@@ -1513,7 +1486,7 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {
{name: "all digest containers updated", remainingImage: "new-image", wantPending: false},
} {
t.Run(tt.name, func(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
imageRef := "registry.example.com/team/app:1.2.3"
record := imageupdate.ImageUpdateRecord{ID: "shared-image", Repository: "registry.example.com/team/app", Tag: "1.2.3", HasUpdate: true, UpdateType: imageupdate.UpdateTypeDigest}
require.NoError(t, db.Create(&record).Error)
@@ -1522,12 +1495,14 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {
{ID: "remaining", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: tt.remainingImage, State: container.StateRunning},
{ID: "separate-tag-policy", Image: imageRef, ImageID: "old-image", State: container.StateRunning, Labels: map[string]string{labels.LabelUpdateStrategy: "tag"}},
}
- server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})
- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})
+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
- converted := imageUpdateRecordToModuleInternal(record)
- converted.ContainerID = "updated"
+ converted := updater.ImageUpdateRecord{
+ ID: record.ID, ContainerID: "updated", Repository: record.Repository, Tag: record.Tag,
+ HasUpdate: record.HasUpdate, UpdateType: updater.UpdateType(record.UpdateType),
+ }
require.NoError(t, svc.ClearImageUpdateRecord(t.Context(), converted))
var actual imageupdate.ImageUpdateRecord
require.NoError(t, db.First(&actual, "id = ?", record.ID).Error)
@@ -1536,50 +1511,33 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {
}
}
-type projectCheckRegistryInternal struct{}
+type projectCheckRegistry struct{}
-func (projectCheckRegistryInternal) ListTags(context.Context, string) ([]string, error) {
+func (projectCheckRegistry) ListTags(context.Context, string) ([]string, error) {
return []string{"3.20.0", "3.20.1", "3.20.2"}, nil
}
-func (projectCheckRegistryInternal) ImageDigest(context.Context, string) (string, error) {
+func (projectCheckRegistry) ImageDigest(context.Context, string) (string, error) {
return "sha256:" + strings.Repeat("a", 64), nil
}
-func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {
+func TestUpdaterProjectChecksWithoutContainers(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !assert.Contains(t, r.URL.Path, "/images/", "project checks must not need containers") {
return
}
w.Header().Set("Content-Type", "application/json")
- if !assert.NoError(
- t,
- json.MarshalWrite(
- w,
- dockertypesimage.InspectResponse{
- ID: "sha256:" + strings.Repeat(
- "a",
- 64,
- ),
- RepoDigests: []string{"alpine@sha256:" + strings.Repeat(
- "a",
- 64,
- )},
- },
- ),
- ) {
+ digest := "sha256:" + strings.Repeat("a", 64)
+ if !assert.NoError(t, json.MarshalWrite(w, dockertypesimage.InspectResponse{ID: digest, RepoDigests: []string{"alpine@" + digest}})) {
return
}
}))
defer server.Close()
- puller := &fakeImagePullerInternal{}
+ puller := &fakeImagePuller{}
engine, err := updater.New(updater.Config{
- DockerClientProvider: fakeDockerClientProviderInternal{client: newTestDockerClientInternal(
- t,
- server,
- )},
- RegistryTagLister: projectCheckRegistryInternal{},
- RegistryDigestResolver: projectCheckRegistryInternal{},
+ DockerClientProvider: fakeDockerClientProvider{client: newTestDockerClient(t, server)},
+ RegistryTagLister: projectCheckRegistry{},
+ RegistryDigestResolver: projectCheckRegistry{},
ImagePuller: puller,
})
require.NoError(t, err)
@@ -1595,23 +1553,16 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {
} {
localConfig := types.ServiceConfig{Name: tt.name, Image: "alpine:3.20.0", Labels: map[string]string{labels.LabelUpdateConstraint: tt.constraint}}
if metadata {
- model, loadErr := projects.LoadComposeProjectFromContent(
- t.Context(),
- projecttypes.ComposeContentOptions{
- ProjectName: "metadata",
- WorkingDir: t.TempDir(),
- ComposeContent: fmt.Sprintf(
- "x-arcane:\n updater:\n strategy: auto\nservices:\n %s:\n image: alpine:3.20.0\n x-arcane:\n "+
- " updater:\n constraint: %q\n",
- tt.name,
- tt.constraint,
- ),
- },
- )
+ compose := "x-arcane:\n updater:\n strategy: auto\nservices:\n %s:\n image: alpine:3.20.0\n x-arcane:\n updater:\n constraint: %q\n"
+ model, loadErr := projects.LoadComposeProjectFromContent(t.Context(), projecttypes.ComposeContentOptions{
+ ProjectName: "metadata",
+ WorkingDir: t.TempDir(),
+ ComposeContent: fmt.Sprintf(compose, tt.name, tt.constraint),
+ })
require.NoError(t, loadErr)
localConfig = model.Services[tt.name]
}
- record := service.checkProjectServiceInternal(t.Context(), "project", localConfig)
+ record := service.checkProjectService(t.Context(), "project", localConfig)
require.Nil(t, record.LastError)
require.Equal(t, tt.available, record.HasUpdate)
require.Equal(t, tt.target, *record.LatestVersion)
@@ -1623,7 +1574,7 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {
}
func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {
- db := setupProjectTestDBInternal(t)
+ db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}, &AutoUpdateRecord{}))
pool, err := db.DB.DB()
require.NoError(t, err)
@@ -1637,22 +1588,14 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {
return
}
case strings.HasSuffix(r.URL.Path, "/containers/app/json"):
- if !assert.NoError(
- t,
- json.MarshalWrite(
- w,
- container.InspectResponse{
- ID: "app",
- Name: "/app",
- Image: "sha256:baseline",
- Config: &container.Config{Image: immutableRef},
- State: &container.State{
- StartedAt: "baseline",
- Running: true,
- },
- },
- ),
- ) {
+ inspect := container.InspectResponse{
+ ID: "app",
+ Name: "/app",
+ Image: "sha256:baseline",
+ Config: &container.Config{Image: immutableRef},
+ State: &container.State{StartedAt: "baseline", Running: true},
+ }
+ if !assert.NoError(t, json.MarshalWrite(w, inspect)) {
return
}
default:
@@ -1661,47 +1604,33 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {
}))
defer server.Close()
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(
- db,
- nil,
- (&docker.DockerClientService{}).WithClient(newTestDockerClientInternal(
- t,
- server,
- )),
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- activityService,
- nil,
- nil,
- nil,
- nil,
- )
+ dockerSvc := (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server))
+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, err)
- runtime := francistest.New(t)
- require.NoError(t, svc.RegisterActors(runtime))
- francistest.Start(t, runtime)
- require.NoError(t, svc.Start(t.Context()))
- t.Cleanup(func() {
- ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), time.Second)
- defer cancel()
- require.NoError(t, svc.Stop(ctx))
- })
+ harness := flowtest.New(t, activityService)
+ require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
accepted, err := svc.AcceptSingleContainerUpdate(t.Context(), "app")
require.NoError(t, err)
require.Eventually(t, func() bool {
- var state arcaneupdater.SingleUpdateState
- if runtime.Service().GetState(t.Context(), execution.StateType, accepted.ID, &state) != nil || state.Status != "completed" {
- return false
- }
detail, getActivityDetailErr := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)
return getActivityDetailErr == nil && detail.Activity.Status == activitytypes.StatusSuccess
- }, 3*time.Second, 10*time.Millisecond)
+ }, 5*time.Second, 10*time.Millisecond)
detail, err := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)
require.NoError(t, err)
require.Equal(t, activitytypes.StatusSuccess, detail.Activity.Status)
}
+
+func TestUpdaterService_WorkflowDefinitionsAreVersioned(t *testing.T) {
+ db := setupProjectTestDB(t)
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)
+ require.NoError(t, imageUpdates.RegisterWorkflows(harness.Engine))
+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ require.NoError(t, err)
+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+
+ // auto-update embeds image-check, so bumping image-check also requires bumping auto-update.
+ flowtest.AssertDefinitions(t, harness)
+}
diff --git a/backend/internal/upload/handler.go b/backend/internal/upload/handler.go
index d2675980f..476e48bb7 100644
--- a/backend/internal/upload/handler.go
+++ b/backend/internal/upload/handler.go
@@ -146,12 +146,7 @@ func (h *UploadHandler) DeleteSession(ctx context.Context, input *DeleteUploadSe
}
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Upload session deleted"},
- },
- }, nil
+ return handlerutil.MessageOutput("Upload session deleted", ""), nil
}
// LegacyMultipartMiddleware converts a deprecated multipart/form-data request
diff --git a/backend/internal/user/handler.go b/backend/internal/user/handler.go
index 1f2487695..52775e2a6 100644
--- a/backend/internal/user/handler.go
+++ b/backend/internal/user/handler.go
@@ -279,14 +279,7 @@ func (h *UserHandler) DeleteUser(ctx context.Context, input *DeleteUserInput) (*
h.invalidateUserTokenCache(input.UserID)
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "User deleted successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("User deleted successfully", ""), nil
}
// GetUserAvatar returns the custom profile picture for a usertypes.
diff --git a/backend/internal/variable/handler.go b/backend/internal/variable/handler.go
index 161413cb0..f4ecb62e1 100644
--- a/backend/internal/variable/handler.go
+++ b/backend/internal/variable/handler.go
@@ -171,14 +171,7 @@ func (h *VariableHandler) UpdateMaterializedVariables(ctx context.Context, input
return nil, huma.Error500InternalServerError("Failed to update global variables: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Global variables updated successfully",
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Global variables updated successfully", ""), nil
}
func variableMutationHTTPErrorInternal(err error) error {
diff --git a/backend/internal/volume/children/backup/children/policies/service.go b/backend/internal/volume/children/backup/children/policies/service.go
index 46ff57cb5..c64497942 100644
--- a/backend/internal/volume/children/backup/children/policies/service.go
+++ b/backend/internal/volume/children/backup/children/policies/service.go
@@ -5,23 +5,25 @@ import (
"errors"
"fmt"
"log/slog"
+ "path"
+ "slices"
"strings"
+ "time"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
- backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/volume"
- "go.getarcane.app/kit/pkg"
+ "github.com/italypaleale/francis/builtin/workflow"
"gorm.io/gorm"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
)
@@ -33,7 +35,6 @@ type Dependencies struct {
DB *database.DB
S3Destinations *s3.S3DestinationService
Settings *settings.SettingsService
- Activity *activity.ActivityService
AlreadyRunning error
LatestRun func(ctx context.Context, policyID string) (*volume.BackupEntry, error)
CreateBackup func(ctx context.Context, volumeName, policyID string) (*volume.Backup, error)
@@ -42,8 +43,15 @@ type Dependencies struct {
// Service owns per-volume backup policies and their scheduled jobs.
type Service struct {
- deps Dependencies
- jobs *entityjobs.Registry
+ deps Dependencies
+ jobs *entityjobs.Registry
+ flow *flow.Engine
+ workflow *flow.Workflow
+}
+
+// scheduledBackupInput is the policy a scheduled backup job runs for.
+type scheduledBackupInput struct {
+ PolicyID string `json:"policyId"`
}
func NewService(deps Dependencies) *Service {
@@ -56,13 +64,32 @@ func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.D
return s.jobs.SetScheduler(ctx, dynamicScheduler, admissionGate)
}
-func (s *Service) policiesInternal(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {
+// RegisterWorkflows defines the scheduled backup workflow while the host is still unstarted.
+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
+ var err error
+ s.flow = engine
+ s.workflow, err = engine.Define(flow.Definition{
+ Name: "volume-backup-policy",
+ Version: 1,
+ Fingerprint: "cbf39d2f46876dd35ec9c1a792e7946097827079c8fc33c00b0f8034e8044534",
+ Concurrency: 4,
+ Timeout: 24 * time.Hour,
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeResourceAction, ResourceType: new("volume_backup"), StartedBy: &user.SystemUser,
+ Step: "Creating scheduled backup", LatestMessage: "Creating scheduled volume backup",
+ },
+ Labels: map[string]string{"backup": "Creating scheduled backup"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runScheduledBackup), workflow.WithMaxAttempts(1))},
+ })
+ return err
+}
+
+func (s *Service) policies(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {
var policies []VolumeBackupPolicy
if err := s.deps.DB.WithContext(ctx).Where("volume_name = ?", volumeName).Order("created_at ASC").Find(&policies).Error; err != nil {
return nil, fmt.Errorf("failed to load volume backup policies: %w", err)
}
- return policies,
- nil
+ return policies, nil
}
// Policy loads one policy of the volume; an empty or unknown ID returns nil.
@@ -82,19 +109,13 @@ func (s *Service) Policy(ctx context.Context, volumeName, policyID string) (*Vol
}
func (s *Service) GetBackupPolicies(ctx context.Context, volumeName string) (*volume.BackupPolicyCollection, error) {
- policies, err := s.policiesInternal(ctx, volumeName)
+ policies, err := s.policies(ctx, volumeName)
if err != nil {
return nil, err
}
result := &volume.BackupPolicyCollection{Policies: make([]volume.BackupPolicy, 0, len(policies))}
- destinations := make(map[string]backuptypes.S3Destination)
- if s.deps.S3Destinations != nil {
- available, listErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)
- if listErr == nil {
- result.S3Available = len(available) > 0
- destinations = available
- }
- }
+ destinations := s.deps.S3Destinations.DestinationsByID(ctx)
+ result.S3Available = len(destinations) > 0
for i := range policies {
lastRun, runErr := s.deps.LatestRun(ctx, policies[i].ID)
if runErr != nil {
@@ -117,7 +138,7 @@ func (s *Service) GetBackupPolicies(ctx context.Context, volumeName string) (*vo
}
func (s *Service) UpdateBackupPolicies(ctx context.Context, volumeName string, updates []volume.UpdateBackupPolicy) (*volume.BackupPolicyCollection, error) {
- existing, err := s.policiesInternal(ctx, volumeName)
+ existing, err := s.policies(ctx, volumeName)
if err != nil {
return nil, err
}
@@ -140,7 +161,7 @@ func (s *Service) UpdateBackupPolicies(ctx context.Context, volumeName string, u
return nil
},
Unregister: s.jobs.Unregister,
- Reschedule: s.rescheduleInternal,
+ Reschedule: s.reschedule,
}
if runErr := reconcile.Run(ctx, updates); runErr != nil {
return nil, runErr
@@ -158,70 +179,68 @@ func (s *Service) HasEnabledBackupPolicy(ctx context.Context, volumeName string)
return count > 0, nil
}
-func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {
+// runScheduledBackup is the volume-backup-policy step. A delivery that finds the checkpoint
+// it recorded resumes that backup instead of starting another.
+func (s *Service) runScheduledBackup(ctx context.Context, t flow.Task) (any, error) {
+ var input scheduledBackupInput
+ if err := t.Payload(&input); err != nil {
+ return nil, err
+ }
+ // The run belongs to one policy, so its checkpoint is found by kind; it names the volume as the backup did,
+ // even if the volume was renamed since the job was registered.
+ // A retry after a rename can leave an older checkpoint first, so the latest one is the run's own.
+ previous, _ := jobcontext.Run(ctx)
+ for _, target := range slices.Backward(previous.Outcome.Targets) {
+ if target.ResourceType != "volume_backup" || len(target.RecoveryData) == 0 {
+ continue
+ }
+ // A retry of a backup that already failed runs a new one below.
+ if outcome, err := s.deps.Reconcile(ctx, previous, target.ID); !errors.Is(err, backup.ErrBackupSettled) {
+ return outcome, err
+ }
+ break
+ }
var policy VolumeBackupPolicy
- if err := s.deps.DB.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {
+ if err := s.deps.DB.WithContext(ctx).Where("id = ? AND enabled = ?", input.PolicyID, true).First(&policy).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return scheduler.Outcome{Status: scheduler.Canceled, Message: "Backup policy disabled or deleted"}, nil
}
- return scheduler.Outcome{}, err
+ return nil, err
}
- if previous, ok := jobcontext.Run(ctx); ok {
- outcome := jobcontext.ConfirmedTarget(previous, policy.VolumeName)
- if outcome.Status == scheduler.Succeeded {
- return outcome, nil
- }
+ var remoteErr error
+ if policy.S3Enabled {
+ root := path.Join(backup.VolumeRoot, s.deps.Settings.GetSettingsConfig().InstanceID.Value)
+ remoteErr = backup.CheckScheduledRemote(ctx, s.deps.DB, s.deps.S3Destinations, "volume_backups", policy.S3DestinationID, root)
}
- remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, &policy)
- if checkErr != nil {
- return scheduler.Outcome{}, checkErr
+ remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {
+ return backup.DisableStoredRemote(ctx, s.deps.DB, &VolumeBackupPolicy{}, policy.ID, policy.S3DestinationID, policy.LocalEnabled, field)
+ })
+ if disableErr != nil {
+ return nil, disableErr
+ }
+ if remoteDisabled {
+ s.reschedule(ctx, &policy)
}
if remoteDisabled && !policy.LocalEnabled {
return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
}
- var entry *volume.Backup
- activityID, err := activitylib.RunHandlerActivity(ctx, s.deps.Activity, activitylib.HandlerOptions{
- EnvironmentID: "0",
- Type: activitytypes.TypeResourceAction,
- ResourceType: "volume_backup",
- ResourceID: policy.VolumeName,
- ResourceName: policy.VolumeName,
- User: &user.SystemUser,
- Step: "Creating scheduled backup",
- Message: "Creating scheduled volume backup",
- SuccessMessage: "Scheduled volume backup created successfully",
- Metadata: database.JSON{
- "action": "scheduled_volume_backup",
- "policyId": policy.ID,
- "schedule": policy.Schedule,
- "volumeName": policy.VolumeName,
- "retentionCount": policy.RetentionCount,
- "stopContainers": policy.StopContainers,
- "localEnabled": policy.LocalEnabled,
- "s3Enabled": policy.S3Enabled,
- "s3DestinationId": policy.S3DestinationID,
- },
- }, func(activityCtx context.Context) error {
- var backupErr error
- entry, backupErr = s.deps.CreateBackup(activityCtx, policy.VolumeName, policy.ID)
- return backupErr
- })
+ entry, err := s.deps.CreateBackup(ctx, policy.VolumeName, policy.ID)
if errors.Is(err, s.deps.AlreadyRunning) {
slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)
- return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: another backup is running for this volume"}, nil
}
if err != nil {
slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)
- return scheduler.Outcome{}, err
+ return nil, err
}
slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backupId", entry.ID, "remoteSnapshotId", entry.RemoteSnapshotID)
if remoteDisabled {
- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil
+ return scheduler.Outcome{Status: scheduler.Partial, Message: backup.RemoteDisabledMessage}, nil
}
- return scheduler.Outcome{Status: scheduler.Succeeded, ActivityID: activityID}, nil
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Scheduled volume backup created successfully"}, nil
}
-func (s *Service) rescheduleInternal(ctx context.Context, policy *VolumeBackupPolicy) {
+func (s *Service) reschedule(ctx context.Context, policy *VolumeBackupPolicy) {
if policy == nil {
return
}
@@ -229,22 +248,31 @@ func (s *Service) rescheduleInternal(ctx context.Context, policy *VolumeBackupPo
s.jobs.Unregister(ctx, policy.ID)
return
}
- policyID := policy.ID
- s.jobs.Register(ctx, policyID,
- func(ctx context.Context) string {
+ policyID, volumeName := policy.ID, policy.VolumeName
+ s.jobs.Add(ctx, &flow.Job{
+ Engine: s.flow,
+ Workflow: s.workflow,
+ JobName: s.jobs.JobName(policyID),
+ Payload: scheduledBackupInput{PolicyID: policyID},
+ Activity: activitylib.StartRequest{
+ ResourceID: new(volumeName), ResourceName: new(volumeName),
+ Metadata: database.JSON{
+ "action": "scheduled_volume_backup", "policyId": policyID, "schedule": policy.Schedule, "volumeName": volumeName,
+ "retentionCount": policy.RetentionCount, "stopContainers": policy.StopContainers,
+ "localEnabled": policy.LocalEnabled, "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,
+ },
+ },
+ ScheduleFn: func(ctx context.Context) string {
var current VolumeBackupPolicy
if err := s.deps.DB.WithContext(ctx).Where("id = ?", policyID).First(&current).Error; err != nil {
return defaultSchedule
}
return current.Schedule
},
- func(ctx context.Context) (scheduler.Outcome, error) {
- return s.runScheduledBackupInternal(ctx, policyID)
- },
- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
- return s.deps.Reconcile(ctx, previous, policy.VolumeName)
+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
+ return s.deps.Reconcile(ctx, previous, volumeName)
},
- )
+ })
}
func (s *Service) RegisterJobsOnStartup(ctx context.Context) {
@@ -257,14 +285,14 @@ func (s *Service) RegisterJobsOnStartup(ctx context.Context) {
return
}
for i := range policies {
- s.rescheduleInternal(ctx, &policies[i])
+ s.reschedule(ctx, &policies[i])
}
slog.InfoContext(ctx, "Registered scheduled volume backup jobs", "count", len(policies))
}
// Remove unregisters and deletes every policy of a removed volume.
func (s *Service) Remove(ctx context.Context, volumeName string) {
- policies, err := s.policiesInternal(ctx, volumeName)
+ policies, err := s.policies(ctx, volumeName)
if err != nil {
return
}
@@ -280,27 +308,3 @@ func (s *Service) Remove(ctx context.Context, volumeName string) {
func (s *Service) Rename(tx *gorm.DB, oldName, newName string) error {
return tx.Model(&VolumeBackupPolicy{}).Where("volume_name = ?", oldName).Update("volume_name", newName).Error
}
-
-func (s *Service) disableMissingS3Internal(ctx context.Context, policy *VolumeBackupPolicy) (bool, error) {
- if !policy.S3Enabled {
- return false, nil
- }
- err := backup.CheckScheduledRemote(ctx, s.deps.DB, s.deps.S3Destinations, "volume_backups", policy.S3DestinationID, "arcane-volume-backups/"+s.deps.Settings.GetSettingsConfig().InstanceID.Value)
- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
- return false, nil
- }
- column := kit.Ternary(policy.LocalEnabled, "s3_enabled", "enabled")
- result := s.deps.DB.WithContext(ctx).Model(&VolumeBackupPolicy{}).
- Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).
- Update(column, false)
- if result.Error != nil || result.RowsAffected == 0 {
- return false, result.Error
- }
- if policy.LocalEnabled {
- policy.S3Enabled = false
- } else {
- policy.Enabled = false
- }
- s.rescheduleInternal(ctx, policy)
- return true, nil
-}
diff --git a/backend/internal/volume/children/backup/children/policies/service_test.go b/backend/internal/volume/children/backup/children/policies/service_test.go
index 1e2c3227b..389c88412 100644
--- a/backend/internal/volume/children/backup/children/policies/service_test.go
+++ b/backend/internal/volume/children/backup/children/policies/service_test.go
@@ -18,11 +18,11 @@ import (
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func noLatestRunInternal(context.Context, string) (*volume.BackupEntry, error) {
+func noLatestRun(context.Context, string) (*volume.BackupEntry, error) {
return nil, nil
}
-func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {
+func newVolumeAdmissionForTest(t testing.TB) *runs.Admission {
t.Helper()
runtime := francistest.New(t)
gate := runs.NewAdmission(runtime.Service(), t.Name())
@@ -31,21 +31,21 @@ func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {
return gate
}
-type volumeBackupPolicySchedulerInternal struct {
+type volumeBackupPolicyScheduler struct {
submitted []scheduler.Request
jobs map[string]scheduler.Job
}
-func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {
+func (s *volumeBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {
s.jobs[job.Name()] = job
return nil
}
-func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *volumeBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {
delete(s.jobs, name)
}
-func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {
+func (s *volumeBackupPolicyScheduler) HasJob(name string) bool {
_, ok := s.jobs[name]
return ok
}
@@ -55,9 +55,9 @@ func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testing
require.NoError(t, err)
require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))
db := &database.DB{DB: gormDB}
- jobScheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}
- service := NewService(Dependencies{DB: db, LatestRun: noLatestRunInternal})
- gate := newVolumeAdmissionForTestInternal(t)
+ jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}
+ service := NewService(Dependencies{DB: db, LatestRun: noLatestRun})
+ gate := newVolumeAdmissionForTest(t)
require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))
collection, err := service.UpdateBackupPolicies(t.Context(), "app-data", []volume.UpdateBackupPolicy{
@@ -131,7 +131,7 @@ func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {
service := NewService(Dependencies{
DB: db,
S3Destinations: s3.NewS3DestinationService(db, nil),
- LatestRun: noLatestRunInternal,
+ LatestRun: noLatestRun,
})
collection, err := service.UpdateBackupPolicies(t.Context(), "app-data", []volume.UpdateBackupPolicy{{
Schedule: "0 0 2 * * *",
@@ -156,13 +156,13 @@ func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {
require.Equal(t, destination.ID, stored.S3DestinationID)
}
-func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {
+func TestBackupPolicyDestinationLookupFailure(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&VolumeBackupPolicy{}, &s3.S3Destination{}))
policy := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 2 * * *", S3DestinationID: "missing"}
require.NoError(t, db.Create(policy).Error)
- service := NewService(Dependencies{DB: &database.DB{DB: db}, S3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil), LatestRun: noLatestRunInternal})
+ service := NewService(Dependencies{DB: &database.DB{DB: db}, S3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil), LatestRun: noLatestRun})
for _, drop := range []bool{false, true} {
if drop {
require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))
@@ -176,7 +176,7 @@ func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {
}
}
-func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
+func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
s.submitted = append(s.submitted, request)
return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil
}
diff --git a/backend/internal/volume/children/backup/children/restore/handler.go b/backend/internal/volume/children/backup/children/restore/handler.go
index 5306408f7..d7cf5201b 100644
--- a/backend/internal/volume/children/backup/children/restore/handler.go
+++ b/backend/internal/volume/children/backup/children/restore/handler.go
@@ -3,14 +3,12 @@ package restore
import (
"context"
"errors"
- "strings"
"github.com/danielgtaylor/huma/v2"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/base"
uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload"
- "github.com/samber/mo"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -78,12 +76,7 @@ func (h *Handler) RestoreBackup(ctx context.Context, input *RestoreBackupInput)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Restore initiated successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Restore initiated successfully", activityID), nil
}
func (h *Handler) RestoreBackupFiles(ctx context.Context, input *RestoreBackupFilesInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -120,12 +113,7 @@ func (h *Handler) RestoreBackupFiles(ctx context.Context, input *RestoreBackupFi
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Restore initiated successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Restore initiated successfully", activityID), nil
}
func (h *Handler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreInput) (*handlerutil.Out[base.MessageResponse], error) {
@@ -164,10 +152,5 @@ func (h *Handler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreI
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Backup uploaded and restored successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Backup uploaded and restored successfully", activityID), nil
}
diff --git a/backend/internal/volume/children/backup/handler.go b/backend/internal/volume/children/backup/handler.go
index e8d0fecbc..3f9bbd1b6 100644
--- a/backend/internal/volume/children/backup/handler.go
+++ b/backend/internal/volume/children/backup/handler.go
@@ -13,6 +13,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/samber/mo"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -136,19 +137,14 @@ func (h *Handler) ListBackups(ctx context.Context, input *ListBackupsInput) (*Li
return nil, huma.Error500InternalServerError(err.Error())
}
- warning := h.service.backupMountWarningInternal(ctx)
+ warning := h.service.backupMountWarning(ctx)
return &ListBackupsOutput{
Body: VolumeBackupPaginatedResponse{
Success: true,
Data: backups,
Pagination: handlerutil.PaginationResponse(paginationResp),
- Warnings: func() []string {
- if warning == "" {
- return nil
- }
- return []string{warning}
- }(),
+ Warnings: kit.Ternary(warning == "", nil, []string{warning}),
},
}, nil
}
@@ -158,11 +154,7 @@ func (h *Handler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*
if err != nil {
return nil, err
}
- request := volume.CreateBackupRequest{}
- if input.Body != nil {
- request = *input.Body
- }
- entry, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, request)
+ entry, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, kit.FromPtr(input.Body))
if errors.Is(err, h.service.deps.AlreadyRunning) {
return nil, huma.Error409Conflict(err.Error())
}
@@ -197,12 +189,7 @@ func (h *Handler) DeleteBackup(ctx context.Context, input *DeleteBackupInput) (*
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{Message: "Backup deleted successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},
- },
- }, nil
+ return handlerutil.MessageOutput("Backup deleted successfully", activityID), nil
}
func (h *Handler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBackupsInput) (*DiscoverVolumeBackupsOutput, error) {
@@ -225,9 +212,6 @@ func (h *Handler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBack
}
func (h *Handler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*handlerutil.Out[*volume.Backup], error) {
- if h.service == nil {
- return nil, huma.Error500InternalServerError("service not available")
- }
user, _ := userctx.CurrentUserFromContext(ctx)
var uploaded *volume.Backup
runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.go
index 1fbb97a39..13d82fede 100644
--- a/backend/internal/volume/children/backup/helpers.go
+++ b/backend/internal/volume/children/backup/helpers.go
@@ -1,185 +1,105 @@
package backup
import (
+ "cmp"
"context"
"errors"
"fmt"
"log/slog"
- "path"
+ "slices"
"strings"
"time"
- "github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
- "github.com/samber/mo"
"go.getarcane.app/docker"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
)
-type backupStorageMode string
-
const (
- // backupStorageModeArcaneMount means backup helpers mirror an existing Arcane
- // container mount at /backups. This intentionally covers any mount the Arcane
- // container already has at /backups, not exclusively bind mounts.
- backupStorageModeArcaneMount backupStorageMode = "arcane_mount"
- // backupStorageModeNamedVolumeFallback means no suitable Arcane container
- // mount was found, so Arcane's dedicated named backup volume is used.
- backupStorageModeNamedVolumeFallback backupStorageMode = "named_volume_fallback"
-
backupMountMissingWarning = "No volume is mounted at /backups in the Arcane container. Backups will only live inside Docker unless you mount a host path."
volumeBackupContainerRecoveryTimeout = 30 * time.Second
volumeBackupContainerRecoveryInterval = 500 * time.Millisecond
- volumeRusticRepositoryPath = "/repository/volumes"
- localVolumeRepositoryID = "volumes:local"
- legacyVolumePasswordSaltInternal = "arcane-volume-backups:"
+ volumeRusticRepositoryPath = "/repository/volumes"
+ localVolumeRepositoryID = "volumes:local"
+ legacyVolumePasswordSalt = "arcane-volume-backups:"
systemRecoverySnapshotLabel = "arcane-system-recovery"
)
-type backupStorageMountInternal struct {
- mode backupStorageMode
- mount mount.Mount
- requiresEnsure bool
-}
-
-func resolveBackupStorageMountFromMountsInternal(ctx context.Context, mounts []container.MountPoint, target string, readOnly bool) mo.Option[backupStorageMountInternal] {
+// resolveBackupStorageMountFromMounts mirrors the Arcane container's /backups mount at target, or returns nil.
+func resolveBackupStorageMountFromMounts(ctx context.Context, mounts []container.MountPoint, target string, readOnly bool) *mount.Mount {
mirroredMount := docker.MountForDestination(mounts, "/backups", target)
if mirroredMount == nil {
- return mo.None[backupStorageMountInternal]()
+ return nil
}
- // MountForDestination only returns non-nil for bind and named volume mounts.
-
if !readOnly && mirroredMount.ReadOnly {
slog.WarnContext(ctx, "volume service: requested writable backup mount but source is read-only; writes may fail")
}
mirroredMount.ReadOnly = readOnly
-
- return mo.Some(backupStorageMountInternal{
- mode: backupStorageModeArcaneMount,
- mount: *mirroredMount,
- })
-}
-
-func (s *Service) resolveBackupStorageMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) backupStorageMountInternal {
- if dockerClient != nil {
- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
- if err != nil {
- slog.WarnContext(ctx, "volume service: failed to inspect arcane container for backup mount resolution, falling back to named volume", "error", err.Error())
- } else if resolved, ok := resolveBackupStorageMountFromMountsInternal(ctx, inspect.Mounts, target, readOnly).Get(); ok {
- return resolved
- }
- }
-
- return backupStorageMountInternal{
- mode: backupStorageModeNamedVolumeFallback,
- mount: mount.Mount{
- Type: mount.TypeVolume,
- Source: s.deps.BackupVolumeName,
- Target: target,
- ReadOnly: readOnly,
- },
- requiresEnsure: true,
- }
-}
-
-func (s *Service) resolveUsableBackupStorageMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (backupStorageMountInternal, error) {
- backupStorage := s.resolveBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
- if backupStorage.requiresEnsure {
- if err := s.ensureBackupVolumeInternal(ctx); err != nil {
- return backupStorageMountInternal{}, err
- }
- }
- return backupStorage, nil
+ return mirroredMount
}
-func backupMountWarningForStorageInternal(storage backupStorageMountInternal) string {
- return kit.Ternary(storage.mode == backupStorageModeArcaneMount, "", backupMountMissingWarning)
-}
-
-func backupMountWarningFromArcaneMountsInternal(ctx context.Context, mounts []container.MountPoint) string {
- backupStorage, ok := resolveBackupStorageMountFromMountsInternal(ctx, mounts, "/backups", true).Get()
- if ok {
- return backupMountWarningForStorageInternal(backupStorage)
- }
-
- // Backward compatibility: historically either /backups or /restores mount
- // suppressed the warning. Preserve that user-visible behavior.
- for _, m := range mounts {
- if m.Destination == "/restores" {
- return ""
- }
+// backupMountWarningFromArcaneMounts warns unless Arcane mounts /backups; a /restores mount also suppresses it for compatibility.
+func backupMountWarningFromArcaneMounts(mounts []container.MountPoint) string {
+ restoresMounted := slices.ContainsFunc(mounts, func(m container.MountPoint) bool { return m.Destination == "/restores" })
+ if docker.MountForDestination(mounts, "/backups", "/backups") != nil || restoresMounted {
+ return ""
}
-
return backupMountMissingWarning
}
-func (s *Service) backupMountWarningInternal(ctx context.Context) string {
+func (s *Service) backupMountWarning(ctx context.Context) string {
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return ""
}
-
// Cannot determine Arcane mount status (e.g. running outside Docker); suppress warning.
inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
if err != nil {
return ""
}
-
- return backupMountWarningFromArcaneMountsInternal(ctx, inspect.Mounts)
+ return backupMountWarningFromArcaneMounts(inspect.Mounts)
}
// StorageMount resolves the backup repository mount shared with system-backup and workspace operations.
+// It mirrors Arcane's own /backups mount and otherwise falls back to the dedicated named backup volume.
func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {
- storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
- if err != nil {
- return mount.Mount{}, err
+ if dockerClient != nil {
+ inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ if err != nil {
+ slog.WarnContext(ctx, "volume service: failed to inspect arcane container for backup mount resolution, falling back to named volume", "error", err.Error())
+ } else if mirrored := resolveBackupStorageMountFromMounts(ctx, inspect.Mounts, target, readOnly); mirrored != nil {
+ return *mirrored, nil
+ }
}
- return storage.mount, nil
-}
-
-func (s *Service) ensureBackupVolumeInternal(ctx context.Context) error {
slog.DebugContext(ctx, "volume service: ensure backup volume", "backupVolume", s.deps.BackupVolumeName)
- dockerClient, err := s.deps.Docker.GetClient(ctx)
+ ensureClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
- return err
+ return mount.Mount{}, err
}
-
- _, err = dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})
- if err != nil {
- _, err = dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{
- Name: s.deps.BackupVolumeName,
- })
- if err != nil {
- return fmt.Errorf("failed to create backup volume: %w", err)
+ if _, inspectErr := ensureClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{}); inspectErr != nil {
+ if _, createErr := ensureClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName}); createErr != nil {
+ return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)
}
}
- return nil
+ return mount.Mount{Type: mount.TypeVolume, Source: s.deps.BackupVolumeName, Target: target, ReadOnly: readOnly}, nil
}
-func (
- s *Service,
-) stopRunningContainersForBackupInternal(
- ctx context.Context,
- dockerClient *client.Client,
- volumeName string,
- user usertypes.Actor,
- refuseArcaneWriters bool,
-) (
- []container.Summary,
- error,
-) {
+func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {
if s.deps.StopContainer == nil || s.deps.StartContainer == nil {
return nil, errors.New("container service is unavailable")
}
@@ -195,10 +115,8 @@ func (
arcaneOwned = append(arcaneOwned, candidate)
continue
}
- // Arcane's own helper containers mount the volume as well, but they are
- // auto-removed when stopped and recreated on demand. Stopping one here
- // would leave nothing for the restart pass to find, failing an
- // otherwise successful backup or restore.
+ // Arcane's helper containers are auto-removed when stopped, so stopping
+ // one would leave nothing for the restart pass and fail the operation.
if strings.EqualFold(candidate.Labels[libarcane.InternalResourceLabel], "true") {
continue
}
@@ -212,16 +130,14 @@ func (
}
}
containerIDs := docker.FilterContainersUsingVolume(eligible, volumeName)
- containersByID := make(map[string]container.Summary, len(eligible))
- for _, candidate := range eligible {
- containersByID[candidate.ID] = candidate
- }
stopped := make([]container.Summary, 0, len(containerIDs))
- for _, containerID := range containerIDs {
- candidate := containersByID[containerID]
- if stopContainerErr := s.deps.StopContainer(ctx, containerID, user); stopContainerErr != nil {
- stillStopped, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
- return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", containerID, stopContainerErr), restartErr)
+ for _, candidate := range eligible {
+ if !slices.Contains(containerIDs, candidate.ID) {
+ continue
+ }
+ if stopContainerErr := s.deps.StopContainer(ctx, candidate.ID, user); stopContainerErr != nil {
+ stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", candidate.ID, stopContainerErr), restartErr)
}
stopped = append(stopped, candidate)
}
@@ -229,7 +145,7 @@ func (
}
//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop
-func (s *Service) startContainersAfterBackupInternal(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {
+func (s *Service) startContainersAfterBackup(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {
recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)
defer cancel()
@@ -305,113 +221,80 @@ func (s *Service) startContainersAfterBackupInternal(ctx context.Context, docker
return nil, nil
}
-func sanitizeBackupPathInternal(input string) (string, error) {
- trimmed := strings.TrimSpace(input)
- if trimmed == "" {
- return "", errors.New("invalid path: empty")
- }
- cleaned := path.Clean(trimmed)
- if cleaned == "." || cleaned == "/" {
- return "", fmt.Errorf("invalid path: %s", input)
- }
- if path.IsAbs(cleaned) {
- cleaned = strings.TrimPrefix(cleaned, "/")
- }
- if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {
- return "", fmt.Errorf("invalid path: %s", input)
- }
- return cleaned, nil
-}
-
-func sanitizeBackupIDInternal(backupID string) (string, error) {
- cleaned, err := sanitizeBackupPathInternal(backupID)
+func backupArchiveFilename(backupID string) (string, error) {
+ name, err := kit.ValidateFileName(backupID)
if err != nil {
return "", fmt.Errorf("invalid backup id: %w", err)
}
- if strings.Contains(cleaned, "/") {
- return "", errors.New("invalid backup id: path separators not allowed")
- }
- return cleaned, nil
+ return name + ".tar.gz", nil
}
-func backupArchiveFilenameInternal(backupID string) (string, error) {
- sanitizedBackupID, err := sanitizeBackupIDInternal(backupID)
- if err != nil {
- return "", err
- }
- return sanitizedBackupID + ".tar.gz", nil
-}
-
-// volumeBackupPasswordInternal returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.
-func (s *Service) volumeBackupPasswordInternal(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {
+// volumeBackupPassword returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.
+func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {
+ legacyPassword := kit.SHA256Hex(legacyVolumePasswordSalt + s.deps.EncryptionKey)
if s.deps.RecoveryKeys == nil {
- return kit.SHA256Hex(legacyVolumePasswordSaltInternal + s.deps.EncryptionKey), nil
+ return legacyPassword, nil
}
key, err := s.deps.RecoveryKeys.Get(ctx)
if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
- return kit.SHA256Hex(legacyVolumePasswordSaltInternal + s.deps.EncryptionKey), nil
+ return legacyPassword, nil
}
if err != nil {
return "", err
}
- for _, repository := range repositories {
- s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)
- }
- return key, nil
-}
-
-// rekeyRepositoryInternal re-keys a legacy repository once per process; one that opens with neither password is left for the caller's operation to create or report.
-func (s *Service) rekeyRepositoryInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) {
if s.deps.Engine == nil {
- return
+ return key, nil
}
- if done, _ := s.rekeyed.Load(repository.ID); done == recoveryKey {
- return
- }
- if _, err := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey); err == nil {
- s.rekeyed.Store(repository.ID, recoveryKey)
- return
- }
- if repository.ID == localVolumeRepositoryID {
- writable, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)
- if err != nil {
- slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", err.Error())
- return
+ // Re-key each legacy repository once per process; one that opens with neither password is left for the caller to create or report.
+ for _, repository := range repositories {
+ if done, _ := s.rekeyed.Load(repository.ID); done == key {
+ continue
}
- repository = writable
- }
- if err := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, kit.SHA256Hex(legacyVolumePasswordSaltInternal+s.deps.EncryptionKey), recoveryKey); err != nil {
- slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", err.Error())
- return
+ if _, listErr := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, key); listErr == nil {
+ s.rekeyed.Store(repository.ID, key)
+ continue
+ }
+ if repository.ID == localVolumeRepositoryID {
+ writable, localErr := s.localRusticRepository(ctx, dockerClient, false)
+ if localErr != nil {
+ slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", localErr.Error())
+ continue
+ }
+ repository = writable
+ }
+ if changeErr := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, legacyPassword, key); changeErr != nil {
+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", changeErr.Error())
+ continue
+ }
+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)
+ s.rekeyed.Store(repository.ID, key)
}
- slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)
- s.rekeyed.Store(repository.ID, recoveryKey)
+ return key, nil
}
-func (s *Service) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
- storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)
+func (s *Service) localRusticRepository(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
+ storage, err := s.StorageMount(ctx, dockerClient, "/repository", readOnly)
if err != nil {
return backup.Repository{}, err
}
return backup.Repository{
ID: localVolumeRepositoryID,
Environment: []string{"RUSTIC_REPOSITORY=" + volumeRusticRepositoryPath},
- Mounts: []mount.Mount{storage.mount},
+ Mounts: []mount.Mount{storage},
}, nil
}
-func (s *Service) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {
- return s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, "")
+// remoteInstanceID resolves the instance whose S3 root holds a backup; empty means this instance.
+func (s *Service) remoteInstanceID(instanceID string) string {
+ return cmp.Or(strings.TrimSpace(instanceID), strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value))
}
-// remoteRusticRepositoryForInstanceInternal addresses one instance's root on the destination; empty means this instance.
-func (s *Service) remoteRusticRepositoryForInstanceInternal(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {
+// remoteRusticRepository addresses one instance's root on the destination; empty instanceID means this instance.
+func (s *Service) remoteRusticRepository(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {
if s.deps.S3Destinations == nil {
- return backup.Repository{}, errors.New("S3 backup service is unavailable")
- }
- if strings.TrimSpace(instanceID) == "" {
- instanceID = strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value)
+ return backup.Repository{}, errors.New("S3 backup destinations are unavailable")
}
+ instanceID = s.remoteInstanceID(instanceID)
if instanceID == "" {
return backup.Repository{}, errors.New("arcane instance ID is unavailable")
}
@@ -421,94 +304,74 @@ func (s *Service) remoteRusticRepositoryForInstanceInternal(ctx context.Context,
}
return backup.Repository{
ID: "volumes:s3:" + destinationID + ":" + instanceID,
- Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),
+ Environment: configuration.RusticEnvironment(backup.VolumeRoot, instanceID),
}, nil
}
-func (s *Service) rusticRepositoryForBackupInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error) {
- if entry.LocalSnapshotID != "" {
- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
- return repository, entry.LocalSnapshotID, err
+// acquireVolumeRun takes the volume's run lease, failing with AlreadyRunning while another backup operation holds it.
+func (s *Service) acquireVolumeRun(ctx context.Context, volumeName string) (*runs.Lease, error) {
+ lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
+ if err != nil {
+ return nil, err
}
- if entry.RemoteSnapshotID != "" {
- repository, err := s.remoteRusticRepositoryForInstanceInternal(ctx, entry.S3DestinationID, entry.RemoteInstanceID)
- return repository, entry.RemoteSnapshotID, err
+ if !admitted {
+ return nil, s.deps.AlreadyRunning
}
- return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")
+ return lease, nil
}
-func (s *Service) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {
- var err error
- if dockerClient == nil {
- dockerClient, err = s.deps.Docker.GetClient(ctx)
- if err != nil {
- return "", nil, err
- }
+// backupDestination names the destination that holds a backup's local and remote copies.
+func backupDestination(local, remote bool) volume.BackupDestination {
+ switch {
+ case local && remote:
+ return volume.BackupDestinationLocalS3
+ case remote:
+ return volume.BackupDestinationS3
+ default:
+ return volume.BackupDestinationLocal
}
+}
- if strings.TrimSpace(helperImage) == "" {
- helperImage, err = s.deps.HelperImage(ctx, dockerClient)
- if err != nil {
- return "", nil, err
- }
+// logBackupEvent records a volume backup event; a nil user logs as the system user.
+func (s *Service) logBackupEvent(ctx context.Context, eventType event.EventType, volumeName string, user *usertypes.Actor, metadata database.JSON) {
+ actor := cmp.Or(user, &usertypes.SystemUser)
+ if err := s.deps.Events.LogVolumeEvent(ctx, eventType, volumeName, volumeName, actor.ID, actor.Username, "0", metadata); err != nil {
+ slog.WarnContext(ctx, "could not log volume backup event", "volume", volumeName, "action", metadata["action"], "error", err)
}
+}
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{"sleep", "infinity"},
- NetworkDisabled: true,
- Labels: volumehelper.Labels(),
+func (s *Service) createBackupTempContainerWithMount(ctx context.Context, dockerClient *client.Client, backupMount mount.Mount) (string, func(), error) {
+ helperImage, err := s.deps.HelperImage(ctx, dockerClient)
+ if err != nil {
+ return "", nil, err
}
-
- hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})
-
resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
+ Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},
+ HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),
})
if err != nil {
return "", nil, fmt.Errorf("failed to create backup temp container: %w", err)
}
-
if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
_, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
return "", nil, fmt.Errorf("failed to start backup temp container: %w", containerStartErr)
}
-
- cleanup := func() {
+ return resp.ID, func() {
_, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
- }
-
- return resp.ID, cleanup, nil
+ }, nil
}
-func (s *Service) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {
+func (s *Service) createBackupTempContainer(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {
slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "readOnly", readOnly)
var err error
if dockerClient == nil {
- dockerClient, err = s.deps.Docker.GetClient(ctx)
- if err != nil {
+ if dockerClient, err = s.deps.Docker.GetClient(ctx); err != nil {
return "", nil, err
}
}
-
- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
+ backupMount, err := s.StorageMount(ctx, dockerClient, target, readOnly)
if err != nil {
return "", nil, err
}
-
- return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)
-}
-
-func volumeSourceMountInternal(volumeName string) mount.Mount {
- return mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume", ReadOnly: true}
-}
-
-func backupDestinationAttemptedInternal(previous scheduler.Run, backupID, destination string) bool {
- for _, evidence := range previous.Outcome.Targets {
- if evidence.ID == backupID+":"+destination {
- return true
- }
- }
- return false
+ return s.createBackupTempContainerWithMount(ctx, dockerClient, backupMount)
}
diff --git a/backend/internal/volume/children/backup/helpers_test.go b/backend/internal/volume/children/backup/helpers_test.go
index 2ab24dbd5..200f8e645 100644
--- a/backend/internal/volume/children/backup/helpers_test.go
+++ b/backend/internal/volume/children/backup/helpers_test.go
@@ -24,19 +24,17 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
)
-func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {
+func TestResolveBackupStorageMountFromMounts(t *testing.T) {
tests := []struct {
name string
mounts []container.MountPoint
target string
readOnly bool
wantResolved bool
- wantMode backupStorageMode
wantType mount.Type
wantSource string
wantTarget string
wantReadOnly bool
- wantEnsure bool
}{
{
name: "mirrors bind mount",
@@ -46,7 +44,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {
target: "/volume",
readOnly: true,
wantResolved: true,
- wantMode: backupStorageModeArcaneMount,
wantType: mount.TypeBind,
wantSource: "/host/backups",
wantTarget: "/volume",
@@ -60,7 +57,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {
target: "/volume",
readOnly: false,
wantResolved: true,
- wantMode: backupStorageModeArcaneMount,
wantType: mount.TypeBind,
wantSource: "/host/backups",
wantTarget: "/volume",
@@ -74,7 +70,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {
target: "/backups",
readOnly: false,
wantResolved: true,
- wantMode: backupStorageModeArcaneMount,
wantType: mount.TypeVolume,
wantSource: "arcane-backups",
wantTarget: "/backups",
@@ -99,40 +94,21 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got, ok := resolveBackupStorageMountFromMountsInternal(t.Context(), tt.mounts, tt.target, tt.readOnly).Get()
- require.Equal(t, tt.wantResolved, ok)
+ got := resolveBackupStorageMountFromMounts(t.Context(), tt.mounts, tt.target, tt.readOnly)
+ require.Equal(t, tt.wantResolved, got != nil)
if !tt.wantResolved {
return
}
- require.Equal(t, tt.wantMode, got.mode)
- require.Equal(t, tt.wantType, got.mount.Type)
- require.Equal(t, tt.wantSource, got.mount.Source)
- require.Equal(t, tt.wantTarget, got.mount.Target)
- require.Equal(t, tt.wantReadOnly, got.mount.ReadOnly)
- require.Equal(t, tt.wantEnsure, got.requiresEnsure)
+ require.Equal(t, tt.wantType, got.Type)
+ require.Equal(t, tt.wantSource, got.Source)
+ require.Equal(t, tt.wantTarget, got.Target)
+ require.Equal(t, tt.wantReadOnly, got.ReadOnly)
})
}
}
-func TestResolveBackupStorageMountInternalFallsBackToNamedVolume(t *testing.T) {
- svc := NewService(Dependencies{BackupVolumeName: "arcane-backups"})
-
- got := svc.resolveBackupStorageMountInternal(t.Context(), nil, "/backups", true)
- require.Equal(t, backupStorageModeNamedVolumeFallback, got.mode)
- require.Equal(t, mount.TypeVolume, got.mount.Type)
- require.Equal(t, "arcane-backups", got.mount.Source)
- require.Equal(t, "/backups", got.mount.Target)
- require.True(t, got.mount.ReadOnly)
- require.True(t, got.requiresEnsure)
-}
-
-func TestBackupMountWarningForStorageInternal(t *testing.T) {
- require.Empty(t, backupMountWarningForStorageInternal(backupStorageMountInternal{mode: backupStorageModeArcaneMount}))
- require.Equal(t, backupMountMissingWarning, backupMountWarningForStorageInternal(backupStorageMountInternal{mode: backupStorageModeNamedVolumeFallback}))
-}
-
-func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {
+func TestBackupMountWarningFromArcaneMounts(t *testing.T) {
tests := []struct {
name string
mounts []container.MountPoint
@@ -184,12 +160,12 @@ func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- require.Equal(t, tt.want, backupMountWarningFromArcaneMountsInternal(t.Context(), tt.mounts))
+ require.Equal(t, tt.want, backupMountWarningFromArcaneMounts(tt.mounts))
})
}
}
-func setupVolumeBackupLifecycleTestInternal(t *testing.T, handler http.Handler) (*Service, *client.Client) {
+func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Service, *client.Client) {
t.Helper()
server := httptest.NewServer(handler)
@@ -235,14 +211,14 @@ func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsin
}
})
- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)
+ stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
require.NoError(t, err)
require.Len(t, stopped, 1)
require.Equal(t, "uses-volume", stopped[0].ID)
- remaining, err := service.startContainersAfterBackupInternal(t.Context(), dockerClient, stopped, actor)
+ remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)
require.NoError(t, err)
require.Empty(t, remaining)
require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)
@@ -279,9 +255,9 @@ func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t
}
})
- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stillStopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)
+ stillStopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
require.ErrorContains(t, err, "failed to stop container second")
require.Empty(t, stillStopped)
require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)
@@ -339,14 +315,14 @@ func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *test
}
})
- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)
+ stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
require.NoError(t, err)
require.Len(t, stopped, 1)
require.Equal(t, "old-id", stopped[0].ID)
- remaining, err := service.startContainersAfterBackupInternal(t.Context(), dockerClient, stopped, actor)
+ remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)
require.NoError(t, err)
require.Empty(t, remaining)
require.Equal(t, []string{"stop:old-id"}, operations)
diff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.go
index 4f080b64e..e567099b4 100644
--- a/backend/internal/volume/children/backup/service.go
+++ b/backend/internal/volume/children/backup/service.go
@@ -9,6 +9,7 @@ import (
"io"
"log/slog"
"path"
+ "slices"
"strings"
"sync"
"time"
@@ -19,18 +20,19 @@ import (
"github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/volume"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
+ kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
userdomain "github.com/getarcaneapp/arcane/backend/v2/internal/user"
@@ -41,6 +43,7 @@ import (
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
@@ -66,7 +69,6 @@ type Dependencies struct {
DB *database.DB
Docker *docker.DockerClientService
Events *event.EventService
- Activity *activity.ActivityService
Settings *settings.SettingsService
Engine *backup.Engine
S3Destinations *s3.S3DestinationService
@@ -89,39 +91,81 @@ type Service struct {
policies *policies.Service
browser *browser.Service
restore *restore.Service
+ flow *flow.Engine
+ // backupWorkflow runs manual backups that StartBackup accepted.
+ backupWorkflow *flow.Workflow
}
-// lockHeldInternal marks a context whose caller already holds the volume's workspace lock.
-type lockHeldInternal struct {
+// backupPlan is one run's resolved destinations; it doubles as the run's recovery checkpoint.
+type backupPlan struct {
+ BackupID string `json:"backupId"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+ S3DestinationID string `json:"s3DestinationId"`
+ Policy *policies.VolumeBackupPolicy `json:"policy,omitempty"`
+}
+
+// manualBackupInput is a manual backup's workflow payload; the row and plan are frozen before it starts.
+type manualBackupInput struct {
+ Checkpoint backupPlan `json:"checkpoint"`
+ VolumeName string `json:"volumeName"`
+ Requester backuptypes.Requester `json:"requester"`
+}
+
+// lockHeld marks a context whose caller already holds the volume's workspace lock.
+type lockHeld struct {
service *Service
volumeName string
}
-type lockHeldKeyInternal struct{}
+type lockHeldKey struct{}
+
+type remoteRepositoryKey struct {
+ destinationID string
+ instanceID string
+}
func NewService(deps Dependencies) *Service {
s := &Service{deps: deps}
+ repository := func(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error) {
+ if entry.LocalSnapshotID != "" {
+ local, err := s.localRusticRepository(ctx, dockerClient, true)
+ return local, entry.LocalSnapshotID, err
+ }
+ if entry.RemoteSnapshotID != "" {
+ remote, err := s.remoteRusticRepository(ctx, entry.S3DestinationID, entry.RemoteInstanceID)
+ return remote, entry.RemoteSnapshotID, err
+ }
+ return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")
+ }
s.policies = policies.NewService(policies.Dependencies{
DB: deps.DB,
S3Destinations: deps.S3Destinations,
Settings: deps.Settings,
- Activity: deps.Activity,
AlreadyRunning: deps.AlreadyRunning,
- LatestRun: s.latestRunInternal,
+ LatestRun: func(ctx context.Context, policyID string) (*volume.BackupEntry, error) {
+ entry, err := s.deps.Store.Latest(ctx, policyID)
+ if err != nil || entry == nil {
+ return nil, err
+ }
+ return new(entry.Entry()), nil
+ },
CreateBackup: func(ctx context.Context, volumeName, policyID string) (*volume.Backup, error) {
return s.CreateBackup(ctx, volumeName, usertypes.SystemUser, backuptypes.VolumeBackupTriggerScheduled, volume.CreateBackupRequest{PolicyID: policyID})
},
Reconcile: s.ReconcileBackup,
})
s.browser = browser.NewService(browser.Dependencies{
- Docker: deps.Docker,
- Engine: deps.Engine,
- Run: deps.Store.Run,
- Repository: s.rusticRepositoryForBackupInternal,
- Password: s.volumeBackupPasswordInternal,
- SanitizePath: sanitizeBackupPathInternal,
- ArchiveFilename: backupArchiveFilenameInternal,
- TempContainer: s.createBackupTempContainerInternal,
+ Docker: deps.Docker,
+ Engine: deps.Engine,
+ Run: deps.Store.Run,
+ Repository: repository,
+ Password: s.volumeBackupPassword,
+ SanitizePath: func(input string) (string, error) {
+ return kit.NormalizeRelativePath(strings.TrimLeft(strings.TrimSpace(input), "/"))
+ },
+ ArchiveFilename: backupArchiveFilename,
+ TempContainer: s.createBackupTempContainer,
Exec: deps.Exec,
})
s.restore = restore.NewService(restore.Dependencies{
@@ -130,24 +174,50 @@ func NewService(deps Dependencies) *Service {
Events: deps.Events,
Locks: deps.Locks,
Run: deps.Store.Run,
- Repository: s.rusticRepositoryForBackupInternal,
- Password: s.volumeBackupPasswordInternal,
- StopContainers: s.stopRunningContainersForBackupInternal,
- StartContainers: s.startContainersAfterBackupInternal,
- SafetyBackup: s.safetyBackupInternal,
+ Repository: repository,
+ Password: s.volumeBackupPassword,
+ StopContainers: s.stopRunningContainersForBackup,
+ StartContainers: s.startContainersAfterBackup,
+ // The safety backup runs under the restore's workspace lock, so executeBackup must not take it again.
+ SafetyBackup: func(ctx context.Context, volumeName string, user usertypes.Actor) (string, error) {
+ ctx = context.WithValue(ctx, lockHeldKey{}, lockHeld{service: s, volumeName: volumeName})
+ request := volume.CreateBackupRequest{Destination: volume.BackupDestinationLocal}
+ entry, err := s.CreateBackup(ctx, volumeName, user, backuptypes.VolumeBackupTriggerSafety, request)
+ if err != nil {
+ return "", err
+ }
+ return entry.ID, nil
+ },
ArchivePaths: s.browser.ArchivePaths,
- ArchiveFilename: backupArchiveFilenameInternal,
+ ArchiveFilename: backupArchiveFilename,
HelperImage: deps.HelperImage,
StorageMount: s.StorageMount,
AcquireHelper: deps.AcquireHelper,
Exec: deps.Exec,
})
- if deps.Engine != nil {
- deps.Engine.RegisterRunKind("volume", s.executeDurableBackupInternal, s.failDurableBackupInternal)
- }
return s
}
+// RegisterWorkflows defines the manual and scheduled backup workflows while the host is still unstarted.
+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {
+ var err error
+ s.flow = engine
+ s.backupWorkflow, err = engine.Define(flow.Definition{
+ Name: "volume-backup",
+ Version: 1,
+ Fingerprint: "8bcb88f11ff66a2b5825101d5bd5f9914f9b76bc56e5d131a82499b819bf9c4a",
+ Concurrency: 4,
+ Timeout: 24 * time.Hour,
+ Activity: activitylib.StartRequest{Type: activitytypes.TypeResourceAction},
+ Labels: map[string]string{"backup": "Creating backup"},
+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},
+ })
+ if err != nil {
+ return err
+ }
+ return s.policies.RegisterWorkflows(engine)
+}
+
// SetScheduler injects the dynamic scheduler and admission gate for per-policy
// backup jobs. Agent mode passes them too: agents run their own volume backups.
func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {
@@ -173,193 +243,93 @@ func (s *Service) RenamePolicies(tx *gorm.DB, oldName, newName string) error {
return s.policies.Rename(tx, oldName, newName)
}
-func entryInternal(entry *volume.Backup) volume.BackupEntry {
- return volume.BackupEntry{
- RemoteAvailable: entry.RemoteAvailable,
- ActivityID: entry.ActivityID,
- ID: entry.ID,
- VolumeName: entry.VolumeName,
- Size: entry.Size,
- CreatedAt: entry.CreatedAt.Format(time.RFC3339),
- Status: entry.Status,
- Trigger: entry.Trigger,
- Destination: entry.Destination,
- Format: entry.Format,
- LocalSnapshotID: entry.LocalSnapshotID,
- RemoteSnapshotID: entry.RemoteSnapshotID,
- S3DestinationID: entry.S3DestinationID,
- S3DestinationName: entry.S3DestinationName,
- PolicyID: entry.PolicyID,
- Error: entry.Error,
- Type: entry.Type,
- }
-}
-
-func (s *Service) latestRunInternal(ctx context.Context, policyID string) (*volume.BackupEntry, error) {
- entry, err := s.deps.Store.Latest(ctx, policyID)
- if err != nil || entry == nil {
- return nil, err
- }
- return new(entryInternal(entry)), nil
-}
-
+// backupEntry converts a backup row to its API entry for policy last runs and accepted manual starts.
func (s *Service) List(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volume.Backup, pagination.Response, error) {
- slog.DebugContext(
- ctx,
- "volume service: list backups paginated",
- "volume",
- volumeName,
- "search",
- params.Search,
- "sort",
- params.Sort,
- "order",
- params.Order,
- "start",
- params.Start,
- "limit",
- params.Limit,
- )
+ slog.DebugContext(ctx, "volume service: list backups paginated", "volume", volumeName, "search", params.Search, "sort", params.Sort,
+ "order", params.Order, "start", params.Start, "limit", params.Limit)
backups, totalItems, err := s.deps.Store.List(ctx, volumeName, params)
if err != nil {
return nil, pagination.Response{}, err
}
- hasS3Destination := false
- for i := range backups {
- if strings.TrimSpace(backups[i].S3DestinationID) != "" {
- hasS3Destination = true
- break
+ if slices.ContainsFunc(backups, func(entry volume.Backup) bool { return strings.TrimSpace(entry.S3DestinationID) != "" }) {
+ destinations := s.deps.S3Destinations.DestinationsByID(ctx)
+ for i := range backups {
+ backups[i].S3DestinationName = destinations[backups[i].S3DestinationID].Name
}
}
- if s.deps.S3Destinations != nil && hasS3Destination {
- // Destination names are decoration; never fail the listing over them.
- destinations, destinationErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)
- if destinationErr != nil {
- slog.WarnContext(ctx, "could not resolve volume backup S3 destination names", "volume", volumeName, "error", destinationErr)
- } else {
- for i := range backups {
- backups[i].S3DestinationName = destinations[backups[i].S3DestinationID].Name
- }
- }
- }
- root := ""
- if s.deps.Settings != nil {
- root = "arcane-volume-backups/" + s.deps.Settings.GetSettingsConfig().InstanceID.Value
- }
- remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.deps.S3Destinations, root)
+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.deps.S3Destinations)
for i := range backups {
- backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, backups[i].RemoteSnapshotID)
+ // Discovered backups live under the instance that wrote them, not necessarily this one.
+ root := ""
+ if s.deps.Settings != nil {
+ root = path.Join(backup.VolumeRoot, s.remoteInstanceID(backups[i].RemoteInstanceID))
+ }
+ backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, root, backups[i].RemoteSnapshotID)
}
return backups, pagination.BuildResponse(totalItems, totalItems, params), nil
}
-// backupPlanInternal is the resolved destination selection for one backup run.
-type backupPlanInternal struct {
- policy *policies.VolumeBackupPolicy
- localEnabled bool
- s3Enabled bool
- s3DestinationID string
- destination volume.BackupDestination
-}
-
-func (
- s *Service,
-) resolveBackupPlanInternal(
- ctx context.Context,
- volumeName string,
- trigger string,
- request volume.CreateBackupRequest,
- suppliedPolicy *policies.VolumeBackupPolicy,
-) (
- backupPlanInternal,
- error,
-) {
+func (s *Service) resolveBackupPlan(ctx context.Context, volumeName, trigger string, request volume.CreateBackupRequest, policy *policies.VolumeBackupPolicy) (backupPlan, error) {
destination := request.Destination
- if destination != "" && destination != volume.BackupDestinationLocal && destination != volume.BackupDestinationS3 && destination != volume.BackupDestinationLocalS3 {
- return backupPlanInternal{}, errors.New("invalid volume backup destination")
+ validDestinations := []volume.BackupDestination{volume.BackupDestinationLocal, volume.BackupDestinationS3, volume.BackupDestinationLocalS3}
+ if destination != "" && !slices.Contains(validDestinations, destination) {
+ return backupPlan{}, errors.New("invalid volume backup destination")
}
- policy := suppliedPolicy
if trigger != backuptypes.VolumeBackupTriggerSafety && policy == nil {
var err error
- policy, err = s.policies.Policy(ctx, volumeName, request.PolicyID)
- if err != nil {
- return backupPlanInternal{}, err
+ if policy, err = s.policies.Policy(ctx, volumeName, request.PolicyID); err != nil {
+ return backupPlan{}, err
}
if request.PolicyID != "" && policy == nil {
- return backupPlanInternal{}, errors.New("volume backup policy not found")
+ return backupPlan{}, errors.New("volume backup policy not found")
}
}
- localEnabled, s3Enabled, s3DestinationID := true, false, strings.TrimSpace(request.S3DestinationID)
+ plan := backupPlan{LocalEnabled: true, S3DestinationID: strings.TrimSpace(request.S3DestinationID), Policy: policy}
if policy != nil {
- localEnabled, s3Enabled, s3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
+ plan.LocalEnabled, plan.S3Enabled, plan.S3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
}
if trigger == backuptypes.VolumeBackupTriggerSafety {
- localEnabled, s3Enabled = true, false
+ plan.LocalEnabled, plan.S3Enabled = true, false
} else if destination != "" {
- localEnabled = destination != volume.BackupDestinationS3
- s3Enabled = destination != volume.BackupDestinationLocal
+ plan.LocalEnabled = destination != volume.BackupDestinationS3
+ plan.S3Enabled = destination != volume.BackupDestinationLocal
}
- if !localEnabled && !s3Enabled {
- return backupPlanInternal{}, errors.New("select at least one volume backup destination")
+ if !plan.LocalEnabled && !plan.S3Enabled {
+ return backupPlan{}, errors.New("select at least one volume backup destination")
}
- if s3Enabled && strings.TrimSpace(s3DestinationID) == "" {
- return backupPlanInternal{}, errors.New("select an S3 destination for the volume backup")
+ if !plan.S3Enabled {
+ plan.S3DestinationID = ""
+ return plan, nil
}
- if s3Enabled {
- if s.deps.S3Destinations == nil {
- return backupPlanInternal{}, errors.New("S3 backup destinations are unavailable")
- }
- if _, err := s.deps.S3Destinations.Configuration(ctx, s3DestinationID); err != nil {
- return backupPlanInternal{}, errors.New("select a valid S3 destination for the volume backup")
- }
- } else {
- s3DestinationID = ""
+ if strings.TrimSpace(plan.S3DestinationID) == "" {
+ return backupPlan{}, errors.New("select an S3 destination for the volume backup")
+ }
+ if s.deps.S3Destinations == nil {
+ return backupPlan{}, errors.New("S3 backup destinations are unavailable")
}
- switch {
- case localEnabled && s3Enabled:
- destination = volume.BackupDestinationLocalS3
- case s3Enabled:
- destination = volume.BackupDestinationS3
- default:
- destination = volume.BackupDestinationLocal
+ if _, err := s.deps.S3Destinations.Configuration(ctx, plan.S3DestinationID); err != nil {
+ return backupPlan{}, errors.New("select a valid S3 destination for the volume backup")
}
- return backupPlanInternal{policy: policy, localEnabled: localEnabled, s3Enabled: s3Enabled, s3DestinationID: s3DestinationID, destination: destination}, nil
+ return plan, nil
}
-func (s *Service) CreateBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger string, request volume.CreateBackupRequest) (_ *volume.Backup, err error) {
+func (s *Service) CreateBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger string, request volume.CreateBackupRequest) (*volume.Backup, error) {
trigger = cmp.Or(trigger, backuptypes.VolumeBackupTriggerManual)
- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, request, nil)
+ plan, err := s.resolveBackupPlan(ctx, volumeName, trigger, request, nil)
if err != nil {
return nil, err
}
- return s.createBackupInternal(ctx, volumeName, user, trigger, request.PolicyID, plan)
-}
-
-// safetyBackupInternal takes the local pre-restore backup for a caller that already holds the volume's workspace lock.
-func (s *Service) safetyBackupInternal(ctx context.Context, volumeName string, user usertypes.Actor) (string, error) {
- ctx = context.WithValue(ctx, lockHeldKeyInternal{}, lockHeldInternal{service: s, volumeName: volumeName})
- entry, err := s.CreateBackup(ctx, volumeName, user, backuptypes.VolumeBackupTriggerSafety, volume.CreateBackupRequest{Destination: volume.BackupDestinationLocal})
+ entry, lease, err := s.prepareBackup(ctx, volumeName, trigger, request.PolicyID, &plan)
if err != nil {
- return "", err
+ return nil, err
}
- return entry.ID, nil
+ defer lease.Release(ctx)
+ return entry, s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, user, plan))
}
// CreateSystemManagedBackup runs the existing volume backup workflow with a transient centralized policy.
-func (
- s *Service,
-) CreateSystemManagedBackup(
- ctx context.Context,
- volumeName string,
- user usertypes.Actor,
- trigger string,
- policyID string,
- policy backuptypes.UpdateBackupPolicy,
-) (
- *volume.Backup,
- error,
-) {
+func (s *Service) CreateSystemManagedBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger, policyID string, policy backuptypes.UpdateBackupPolicy) (*volume.Backup, error) {
if !strings.HasPrefix(policyID, backuptypes.SystemVolumePolicyPrefix) {
return nil, errors.New("invalid system-managed volume backup policy id")
}
@@ -370,44 +340,28 @@ func (
S3DestinationID: policy.S3DestinationID,
}
transient.ID = policyID
- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)
+ plan, err := s.resolveBackupPlan(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)
if err != nil {
return nil, err
}
- return s.createBackupInternal(ctx, volumeName, user, trigger, policyID, plan)
-}
-
-func (s *Service) createBackupInternal(ctx context.Context, volumeName string, user usertypes.Actor, trigger, policyID string, plan backupPlanInternal) (*volume.Backup, error) {
- entry, lease, err := s.prepareBackupInternal(ctx, volumeName, trigger, policyID, plan)
+ entry, lease, err := s.prepareBackup(ctx, volumeName, trigger, policyID, &plan)
if err != nil {
return nil, err
}
defer lease.Release(ctx)
- err = s.executeBackupInternal(ctx, entry, user, plan)
- err = s.completeBackupInternal(ctx, entry, err)
- return entry, err
+ return entry, s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, user, plan))
}
-type volumeBackupRecoveryInternal struct {
- BackupID string `json:"backupId"`
- LocalEnabled bool `json:"localEnabled"`
- S3Enabled bool `json:"s3Enabled"`
- S3DestinationID string `json:"s3DestinationId"`
- Policy *policies.VolumeBackupPolicy `json:"policy,omitempty"`
-}
-
-func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, trigger, policyID string, plan backupPlanInternal) (*volume.Backup, *runs.Lease, error) {
- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
+// prepareBackup admits the run, persists its row, and records plan (now carrying the row ID) as the checkpoint.
+func (s *Service) prepareBackup(ctx context.Context, volumeName, trigger, policyID string, plan *backupPlan) (*volume.Backup, *runs.Lease, error) {
+ lease, err := s.acquireVolumeRun(ctx, volumeName)
if err != nil {
return nil, nil, err
}
- if !admitted {
- return nil, nil, s.deps.AlreadyRunning
- }
entry := &volume.Backup{
VolumeName: volumeName, CreatedAt: time.Now(), Status: backuptypes.VolumeBackupStatusRunning,
- Trigger: trigger, Destination: plan.destination, Format: volume.BackupFormatRustic,
- S3DestinationID: plan.s3DestinationID, PolicyID: policyID,
+ Trigger: trigger, Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled), Format: volume.BackupFormatRustic,
+ S3DestinationID: plan.S3DestinationID, PolicyID: policyID,
Type: backuptypes.ManagementTypeForPolicy(policyID),
}
entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
@@ -415,15 +369,8 @@ func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, trigger
lease.Release(ctx)
return nil, nil, createBackupErr
}
- checkpoint, err := json.Marshal(
- volumeBackupRecoveryInternal{
- BackupID: entry.ID,
- LocalEnabled: plan.localEnabled,
- S3Enabled: plan.s3Enabled,
- S3DestinationID: plan.s3DestinationID,
- Policy: plan.policy,
- },
- )
+ plan.BackupID = entry.ID
+ checkpoint, err := json.Marshal(*plan)
if err == nil {
err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "volume_backup", ID: volumeName, Status: scheduler.Running, RecoveryData: checkpoint})
}
@@ -434,7 +381,12 @@ func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, trigger
return entry, lease, nil
}
-func (s *Service) completeBackupInternal(ctx context.Context, entry *volume.Backup, err error) error {
+func (s *Service) completeBackup(ctx context.Context, entry *volume.Backup, err error) error {
+ // A workflow task stopped by shutdown, a plain cancel, is handed back, so its row stays running for the resumed
+ // task to finish; a user cancel, a workflow that ended or timed out, or work outside a workflow records the failure.
+ if _, resumable := jobcontext.Run(ctx); resumable && errors.Is(context.Cause(ctx), context.Canceled) {
+ return ctx.Err()
+ }
if err == nil {
err = ctx.Err()
}
@@ -452,28 +404,26 @@ func (s *Service) completeBackupInternal(ctx context.Context, entry *volume.Back
return err
}
-func (s *Service) executeBackupInternal(ctx context.Context, entry *volume.Backup, user usertypes.Actor, plan backupPlanInternal) (err error) {
+func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, user usertypes.Actor, plan backupPlan) (err error) {
defer utils.RecoverToError(&err, "volume backup")
- volumeName, trigger := entry.VolumeName, entry.Trigger
- lockHeld, _ := ctx.Value(lockHeldKeyInternal{}).(lockHeldInternal)
- if lockHeld.service != s || lockHeld.volumeName != volumeName {
+ volumeName := entry.VolumeName
+ if held, _ := ctx.Value(lockHeldKey{}).(lockHeld); held.service != s || held.volumeName != volumeName {
defer s.deps.Locks.Lock(volumeName)()
}
- if cancellationErr := ctx.Err(); cancellationErr != nil {
- return cancellationErr
+ if cancelErr := ctx.Err(); cancelErr != nil {
+ return cancelErr
}
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return err
}
+ // stopped holds the containers still awaiting a restart.
var stopped []container.Summary
- containersStopped := false
- if plan.policy != nil && plan.policy.StopContainers {
- stopped, err = s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, false)
- containersStopped = len(stopped) > 0
+ if plan.Policy != nil && plan.Policy.StopContainers {
+ stopped, err = s.stopRunningContainersForBackup(ctx, dockerClient, volumeName, user, false)
defer func() {
- if containersStopped {
- _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ if len(stopped) > 0 {
+ _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)
err = errors.Join(err, restartErr)
}
}()
@@ -481,146 +431,94 @@ func (s *Service) executeBackupInternal(ctx context.Context, entry *volume.Backu
return err
}
}
- if plan.localEnabled && entry.LocalSnapshotID == "" {
- if createLocalBackupSnapshotErr := s.createLocalBackupSnapshotInternal(ctx, dockerClient, entry); createLocalBackupSnapshotErr != nil {
- return createLocalBackupSnapshotErr
- }
- }
- if plan.s3Enabled && entry.RemoteSnapshotID == "" {
- if createRemoteBackupSnapshotErr := s.createRemoteBackupSnapshotInternal(ctx, dockerClient, entry, plan); createRemoteBackupSnapshotErr != nil {
- return createRemoteBackupSnapshotErr
- }
+ if snapshotErr := s.createSnapshots(ctx, dockerClient, entry, plan); snapshotErr != nil {
+ return snapshotErr
}
- if containersStopped {
- stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
- containersStopped = len(stopped) > 0
- if err != nil {
+ if len(stopped) > 0 {
+ if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user); err != nil {
return err
}
}
- // Retention failures must not fail the run: the backup itself succeeded,
- // and the deferred status-save would otherwise mark it failed and hide it
- // from ExpiredRunIDs forever.
- if trigger != backuptypes.VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {
- if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount, plan.s3Enabled); retentionErr != nil {
- slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)
+ // Retention failures must not fail the run: the status save would mark the
+ // succeeded backup failed and hide it from ExpiredRunIDs forever.
+ if entry.Trigger != backuptypes.VolumeBackupTriggerSafety && plan.Policy != nil && plan.Policy.RetentionCount > 0 {
+ expired, retentionErr := backup.ExpiredRunIDs(ctx, s.deps.DB, "volume_backups", plan.Policy.ID, plan.Policy.RetentionCount)
+ if retentionErr == nil && len(expired) > 0 {
+ var expiredEntries []*volume.Backup
+ if expiredEntries, retentionErr = s.deps.Store.Runs(ctx, expired); retentionErr == nil {
+ retentionErr = s.deleteRusticBackups(ctx, expiredEntries, nil, plan.S3Enabled)
+ }
+ }
+ if retentionErr != nil {
+ slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.Policy.ID, "error", retentionErr)
}
}
metadata := database.JSON{"action": "backup_create", "backup_id": entry.ID, "size": entry.Size, "destination": entry.Destination}
- if logErr := s.deps.Events.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)
- }
+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, &user, metadata)
return ctx.Err()
}
-func (s *Service) createLocalBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) error {
- repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
- if repoErr != nil {
- return repoErr
+// createSnapshots writes the plan's missing snapshots; S3 replicates the local snapshot when both are enabled.
+func (s *Service) createSnapshots(ctx context.Context, dockerClient *client.Client, entry *volume.Backup, plan backupPlan) error {
+ source := mount.Mount{Type: mount.TypeVolume, Source: entry.VolumeName, Target: "/volume", ReadOnly: true}
+ input := backup.RootSnapshotInput(source, backup.RunSnapshotTag(entry.ID))
+ if plan.LocalEnabled && entry.LocalSnapshotID == "" {
+ repository, err := s.localRusticRepository(ctx, dockerClient, false)
+ if err != nil {
+ return err
+ }
+ password, err := s.volumeBackupPassword(ctx, dockerClient, repository)
+ if err != nil {
+ return err
+ }
+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Running}); progressErr != nil {
+ return progressErr
+ }
+ snapshot, err := s.deps.Engine.CreateSnapshot(ctx, dockerClient, repository, password, entry.VolumeName, input)
+ if err != nil {
+ return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
+ }
+ entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size
+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {
+ return saveErr
+ }
+ succeeded := scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Succeeded, Message: snapshot.ID}
+ if progressErr := jobcontext.Progress(ctx, succeeded); progressErr != nil {
+ return progressErr
+ }
}
- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
- if passwordErr != nil {
- return passwordErr
+ if !plan.S3Enabled || entry.RemoteSnapshotID != "" {
+ return nil
}
- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Running}); err != nil {
+ remoteRepository, err := s.remoteRusticRepository(ctx, plan.S3DestinationID, "")
+ if err != nil {
return err
}
- localSnapshot, err := s.deps.Engine.CreateSnapshot(
- ctx,
- dockerClient,
- repository,
- password,
- entry.VolumeName,
- backup.RootSnapshotInput(
- volumeSourceMountInternal(
- entry.VolumeName,
- ),
- backup.RunSnapshotTag(
- entry.ID,
- ),
- ),
- )
+ password, err := s.volumeBackupPassword(ctx, dockerClient, remoteRepository)
if err != nil {
- return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
- }
- entry.LocalSnapshotID = localSnapshot.ID
- entry.Size = localSnapshot.Size
- if saveLocalBackupErr := s.deps.Store.Save(ctx, entry); saveLocalBackupErr != nil {
- return saveLocalBackupErr
- }
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_destination",
- ID: entry.ID + ":local",
- Status: scheduler.Succeeded,
- Message: localSnapshot.ID,
- },
- ); progressErr != nil {
- return progressErr
- }
- return nil
-}
-
-func (s *Service) createRemoteBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup, plan backupPlanInternal) error {
- var err error
- remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)
- if repoErr != nil {
- return repoErr
- }
- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)
- if passwordErr != nil {
- return passwordErr
+ return err
}
if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: scheduler.Running}); progressErr != nil {
return progressErr
}
- var remoteSnapshot backup.Snapshot
- if plan.localEnabled {
- localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ var snapshot backup.Snapshot
+ if plan.LocalEnabled {
+ localRepository, localErr := s.localRusticRepository(ctx, dockerClient, true)
if localErr != nil {
return localErr
}
- remoteSnapshot, err = s.deps.Engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
+ snapshot, err = s.deps.Engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
} else {
- remoteSnapshot, err = s.deps.Engine.CreateSnapshot(
- ctx,
- dockerClient,
- remoteRepository,
- password,
- entry.VolumeName,
- backup.RootSnapshotInput(
- volumeSourceMountInternal(
- entry.VolumeName,
- ),
- backup.RunSnapshotTag(
- entry.ID,
- ),
- ),
- )
+ snapshot, err = s.deps.Engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, entry.VolumeName, input)
}
if err != nil {
return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
}
- entry.RemoteSnapshotID = remoteSnapshot.ID
- if entry.Size == 0 {
- entry.Size = remoteSnapshot.Size
+ entry.RemoteSnapshotID, entry.Size = snapshot.ID, cmp.Or(entry.Size, snapshot.Size)
+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {
+ return saveErr
}
- if saveRemoteBackupErr := s.deps.Store.Save(ctx, entry); saveRemoteBackupErr != nil {
- return saveRemoteBackupErr
- }
- if remoteDestinationProgressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "backup_destination",
- ID: entry.ID + ":remote",
- Status: scheduler.Succeeded,
- Message: remoteSnapshot.ID,
- },
- ); remoteDestinationProgressErr != nil {
- return remoteDestinationProgressErr
- }
- return nil
+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: scheduler.Succeeded, Message: snapshot.ID})
}
func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID string) (*volume.Backup, error) {
@@ -628,17 +526,13 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID st
if err != nil {
return nil, err
}
- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)
+ lease, err := s.acquireVolumeRun(ctx, entry.VolumeName)
if err != nil {
return nil, err
}
- if !admitted {
- return nil, s.deps.AlreadyRunning
- }
defer lease.Release(ctx)
// Reload under the lease: a delete may have raced the first read.
- entry, err = s.deps.Store.Run(ctx, backupID)
- if err != nil {
+ if entry, err = s.deps.Store.Run(ctx, backupID); err != nil {
return nil, err
}
if entry.Status != backuptypes.VolumeBackupStatusSucceeded || entry.LocalSnapshotID == "" {
@@ -650,25 +544,19 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID st
if strings.TrimSpace(s3DestinationID) == "" {
return nil, errors.New("select an S3 destination for the upload")
}
- if s.deps.S3Destinations == nil {
- return nil, errors.New("S3 backup destinations are unavailable")
- }
- if _, configurationErr := s.deps.S3Destinations.Configuration(ctx, s3DestinationID); configurationErr != nil {
- return nil, errors.New("select a valid S3 destination for the upload")
- }
- dockerClient, err := s.deps.Docker.GetClient(ctx)
+ remoteRepository, err := s.remoteRusticRepository(ctx, s3DestinationID, "")
if err != nil {
return nil, err
}
- localRepository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return nil, err
}
- remoteRepository, err := s.remoteRusticRepositoryInternal(ctx, s3DestinationID)
+ localRepository, err := s.localRusticRepository(ctx, dockerClient, true)
if err != nil {
return nil, err
}
- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, localRepository, remoteRepository)
+ password, err := s.volumeBackupPassword(ctx, dockerClient, localRepository, remoteRepository)
if err != nil {
return nil, err
}
@@ -676,14 +564,11 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID st
if err != nil {
return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)
}
- entry.RemoteSnapshotID = snapshot.ID
- entry.S3DestinationID = s3DestinationID
- entry.Destination = volume.BackupDestinationLocalS3
+ entry.RemoteSnapshotID, entry.S3DestinationID, entry.Destination = snapshot.ID, s3DestinationID, volume.BackupDestinationLocalS3
if saveUploadedBackupErr := s.deps.Store.Save(ctx, entry); saveUploadedBackupErr != nil {
return nil, fmt.Errorf("failed to save uploaded volume backup: %w", saveUploadedBackupErr)
}
- return entry,
- nil
+ return entry, nil
}
func (s *Service) DeleteBackup(ctx context.Context, backupID string, user *usertypes.Actor) error {
@@ -691,56 +576,93 @@ func (s *Service) DeleteBackup(ctx context.Context, backupID string, user *usert
if err != nil {
return err
}
- // Deletes contend with create/restore/upload on the volume's run lease so
- // a slow operation cannot resurrect or double-forget snapshots. Retention
- // deletes run under CreateBackup's lease and call the internal path.
- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)
+ // Deletes share the volume's run lease with create/restore/upload so a slow
+ // operation cannot resurrect or double-forget snapshots.
+ lease, err := s.acquireVolumeRun(ctx, entry.VolumeName)
if err != nil {
return err
}
- if !admitted {
- return s.deps.AlreadyRunning
- }
defer lease.Release(ctx)
- return s.deleteBackupInternal(ctx, backupID, user)
-}
-
-func (s *Service) deleteBackupInternal(ctx context.Context, backupID string, user *usertypes.Actor) error {
- entry, err := s.deps.Store.Run(ctx, backupID)
- if err != nil {
+ if entry, err = s.deps.Store.Run(ctx, backupID); err != nil {
return err
}
- if entry.Format == volume.BackupFormatArchive {
- return s.deleteArchiveBackupInternal(ctx, entry, user)
+ if entry.Format != volume.BackupFormatArchive {
+ return s.deleteRusticBackups(ctx, []*volume.Backup{entry}, user, true)
}
- return s.deleteRusticBackupsInternal(ctx, []*volume.Backup{entry}, user, true)
-}
-
-type remoteRepositoryKeyInternal struct {
- destinationID string
- instanceID string
+ // Delete the row first: a failed file removal then only leaves an orphan file.
+ if deleteErr := s.deps.Store.Delete(ctx, backupID); deleteErr != nil {
+ return deleteErr
+ }
+ containerID, cleanup, err := s.createBackupTempContainer(ctx, nil, "/volume", false)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backupId", backupID, "error", err.Error())
+ } else {
+ defer cleanup()
+ filename, filenameErr := backupArchiveFilename(backupID)
+ if filenameErr != nil {
+ slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backupId", backupID, "error", filenameErr.Error())
+ } else if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {
+ slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backupId", backupID, "error", err.Error())
+ }
+ }
+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDelete, entry.VolumeName, user, database.JSON{"action": "backup_delete", "backup_id": backupID})
+ return nil
}
-// deleteRusticBackupsInternal forgets snapshots grouped by repository so each repository is pruned once.
-func (s *Service) deleteRusticBackupsInternal(ctx context.Context, entries []*volume.Backup, user *usertypes.Actor, includeRemote bool) error {
+// deleteRusticBackups forgets snapshots grouped by repository so each repository is pruned once.
+func (s *Service) deleteRusticBackups(ctx context.Context, entries []*volume.Backup, user *usertypes.Actor, includeRemote bool) error {
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return err
}
- var localEntries []*volume.Backup
- remoteGroups := make(map[remoteRepositoryKeyInternal][]*volume.Backup)
+ var localSnapshotIDs []string
+ remoteGroups := make(map[remoteRepositoryKey][]*volume.Backup)
for _, entry := range entries {
if entry.LocalSnapshotID != "" {
- localEntries = append(localEntries, entry)
+ localSnapshotIDs = append(localSnapshotIDs, entry.LocalSnapshotID)
}
if includeRemote && entry.RemoteSnapshotID != "" {
- key := remoteRepositoryKeyInternal{destinationID: entry.S3DestinationID, instanceID: entry.RemoteInstanceID}
+ key := remoteRepositoryKey{destinationID: entry.S3DestinationID, instanceID: s.remoteInstanceID(entry.RemoteInstanceID)}
remoteGroups[key] = append(remoteGroups[key], entry)
}
}
- deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, localEntries)
+ var deleteErr error
+ if len(localSnapshotIDs) > 0 {
+ repository, repoErr := s.localRusticRepository(ctx, dockerClient, false)
+ if repoErr == nil {
+ repoErr = s.forgetSnapshots(ctx, dockerClient, repository, localSnapshotIDs)
+ }
+ if repoErr != nil {
+ deleteErr = fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)
+ } else {
+ for _, entry := range entries {
+ entry.LocalSnapshotID = ""
+ }
+ }
+ }
for key, group := range remoteGroups {
- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key.destinationID, key.instanceID, group))
+ snapshotIDs := make([]string, len(group))
+ for index, entry := range group {
+ snapshotIDs[index] = entry.RemoteSnapshotID
+ }
+ repository, repoErr := s.remoteRusticRepository(ctx, key.destinationID, key.instanceID)
+ var observation backuptypes.RepositoryObservation
+ if repoErr == nil {
+ observation, repoErr = backup.CheckRemoteRepository(ctx, s.deps.S3Destinations, key.destinationID, path.Join(backup.VolumeRoot, key.instanceID))
+ }
+ if repoErr == nil && observation.Available {
+ repoErr = s.forgetSnapshots(ctx, dockerClient, repository, snapshotIDs)
+ } else if repoErr == nil {
+ // Nothing remains to forget; drop the references so the entry can be removed.
+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", key.destinationID, "snapshots", len(snapshotIDs))
+ }
+ if repoErr != nil {
+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr))
+ continue
+ }
+ for _, entry := range group {
+ entry.RemoteSnapshotID, entry.S3DestinationID = "", ""
+ }
}
for _, entry := range entries {
if entry.LocalSnapshotID == "" && entry.RemoteSnapshotID == "" {
@@ -748,17 +670,10 @@ func (s *Service) deleteRusticBackupsInternal(ctx context.Context, entries []*vo
deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete volume backup record: %w", deleteBackupErr))
continue
}
- s.logBackupDeleteEventInternal(ctx, entry.VolumeName, entry.ID, user)
+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDelete, entry.VolumeName, user, database.JSON{"action": "backup_delete", "backup_id": entry.ID})
continue
}
- switch {
- case entry.LocalSnapshotID != "" && entry.RemoteSnapshotID != "":
- entry.Destination = volume.BackupDestinationLocalS3
- case entry.LocalSnapshotID != "":
- entry.Destination = volume.BackupDestinationLocal
- default:
- entry.Destination = volume.BackupDestinationS3
- }
+ entry.Destination = backupDestination(entry.LocalSnapshotID != "", entry.RemoteSnapshotID != "")
if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {
deleteErr = errors.Join(deleteErr, saveErr)
}
@@ -772,235 +687,97 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {
if err != nil {
return err
}
- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ repository, err := s.localRusticRepository(ctx, dockerClient, false)
if err != nil {
return err
}
- return s.forgetSnapshotsInternal(ctx, dockerClient, repository, nil)
-}
-
-func (s *Service) forgetLocalSnapshotsInternal(ctx context.Context, dockerClient *client.Client, entries []*volume.Backup) error {
- if len(entries) == 0 {
- return nil
- }
- snapshotIDs := make([]string, len(entries))
- for index, entry := range entries {
- snapshotIDs[index] = entry.LocalSnapshotID
- }
- repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
- if repoErr == nil {
- repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)
- }
- if repoErr != nil {
- return fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)
- }
- for _, entry := range entries {
- entry.LocalSnapshotID = ""
- }
- return nil
-}
-
-func (s *Service) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID, instanceID string, entries []*volume.Backup) error {
- snapshotIDs := make([]string, len(entries))
- for index, entry := range entries {
- snapshotIDs[index] = entry.RemoteSnapshotID
- }
- repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, instanceID)
- if repoErr == nil {
- // Mirror remoteRusticRepositoryForInstanceInternal so the check targets the Rustic root.
- root := instanceID
- if strings.TrimSpace(root) == "" {
- root = strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value)
- }
- var observation backuptypes.RepositoryObservation
- observation, repoErr = backup.CheckRemoteRepository(ctx, s.deps.S3Destinations, destinationID, path.Join("arcane-volume-backups", root))
- switch {
- case repoErr != nil:
- case !observation.Available:
- // Nothing remains to forget; drop the references so the entry can be removed.
- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))
- default:
- repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)
- }
- }
- if repoErr != nil {
- return fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr)
- }
- for _, entry := range entries {
- entry.RemoteSnapshotID = ""
- entry.S3DestinationID = ""
- }
- return nil
+ return s.forgetSnapshots(ctx, dockerClient, repository, nil)
}
-func (s *Service) forgetSnapshotsInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {
- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+func (s *Service) forgetSnapshots(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {
+ password, err := s.volumeBackupPassword(ctx, dockerClient, repository)
if err != nil {
return err
}
return s.deps.Engine.ForgetSnapshots(ctx, dockerClient, repository, password, snapshotIDs)
}
-func (s *Service) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *usertypes.Actor) {
- actingUser := user
- if actingUser == nil {
- actingUser = &usertypes.SystemUser
- }
- if logErr := s.deps.Events.LogVolumeEvent(
- ctx,
- event.EventTypeVolumeBackupDelete,
- volumeName,
- volumeName,
- actingUser.ID,
- actingUser.Username,
- "0",
- database.JSON{
- "action": "backup_delete",
- "backup_id": backupID,
- },
- ); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr)
- }
-}
-
-// DownloadBackup streams a backup as a tar.gz archive. Legacy archive rows
-// stream their stored file; local Rustic rows are restored into a scratch
-// volume and packaged on the fly.
+// DownloadBackup streams a backup as a tar.gz archive: legacy archives stream their stored
+// file, and local Rustic snapshots are restored into a scratch volume and packaged on the fly.
func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *usertypes.Actor) (io.ReadCloser, int64, error) {
entry, err := s.deps.Store.Run(ctx, backupID)
if err != nil {
return nil, 0, err
}
- var reader io.ReadCloser
- var size int64
- if entry.Format == volume.BackupFormatArchive {
- reader, size, err = s.downloadArchiveBackupInternal(ctx, backupID)
- } else {
- reader, size, err = s.downloadRusticBackupInternal(ctx, entry)
- }
- if err != nil {
- return nil, 0, err
- }
- actingUser := user
- if actingUser == nil {
- actingUser = &usertypes.SystemUser
- }
- metadata := database.JSON{"action": "backup_download", "backup_id": backupID, "size": size}
- if logErr := s.deps.Events.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, entry.VolumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup download event", "volume", entry.VolumeName, "error", logErr)
- }
- return reader, size, nil
-}
-
-func (s *Service) downloadRusticBackupInternal(ctx context.Context, entry *volume.Backup) (io.ReadCloser, int64, error) {
- if entry.LocalSnapshotID == "" {
+ if entry.Format != volume.BackupFormatArchive && entry.LocalSnapshotID == "" {
return nil, 0, errors.New("only local volume backups can be downloaded")
}
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return nil, 0, err
}
- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
- if err != nil {
- return nil, 0, err
- }
- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
- if err != nil {
- return nil, 0, err
- }
- scratchVolume := "arcane-rustic-download-" + uuid.New().String()
- if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {
- return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)
- }
- removeScratch := func() {
- _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})
- }
- if restoreSnapshotErr := s.deps.Engine.RestoreSnapshot(
- ctx,
- dockerClient,
- repository,
- password,
- entry.LocalSnapshotID,
- mount.Mount{
- Type: mount.TypeVolume,
- Source: scratchVolume,
- Target: "/volume",
- },
- backup.RestoreOptions{
- DeleteExtra: true,
- },
- ); restoreSnapshotErr != nil {
- removeScratch()
- return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreSnapshotErr)
- }
- helperImage, err := s.deps.HelperImage(ctx, dockerClient)
- if err != nil {
- removeScratch()
- return nil, 0, err
- }
- archiveMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume", ReadOnly: false}
- containerID, removeContainer, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, archiveMount)
- if err != nil {
- removeScratch()
- return nil, 0, err
- }
- cleanup := func() {
- removeContainer()
- removeScratch()
- }
- archivePath := "/tmp/" + entry.ID + ".tar.gz"
- if _, _, execInContainerErr := s.deps.Exec(ctx, containerID, "", []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); execInContainerErr != nil {
- cleanup()
- return nil, 0, fmt.Errorf("failed to package Rustic snapshot for download: %w", execInContainerErr)
- }
- return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)
-}
-
-func (s *Service) downloadArchiveBackupInternal(ctx context.Context, backupID string) (io.ReadCloser, int64, error) {
- filename, err := backupArchiveFilenameInternal(backupID)
- if err != nil {
- return nil, 0, err
+ var containerID, archivePath string
+ var cleanup func()
+ if entry.Format == volume.BackupFormatArchive {
+ filename, filenameErr := backupArchiveFilename(backupID)
+ if filenameErr != nil {
+ return nil, 0, filenameErr
+ }
+ archivePath = path.Join("/volume", filename)
+ containerID, cleanup, err = s.createBackupTempContainer(ctx, dockerClient, "/volume", true)
+ } else {
+ // Restore the local snapshot into a scratch volume and tar it inside a helper container.
+ archivePath = "/tmp/" + entry.ID + ".tar.gz"
+ repository, repoErr := s.localRusticRepository(ctx, dockerClient, true)
+ if repoErr != nil {
+ return nil, 0, repoErr
+ }
+ password, passwordErr := s.volumeBackupPassword(ctx, dockerClient, repository)
+ if passwordErr != nil {
+ return nil, 0, passwordErr
+ }
+ scratchVolume := "arcane-rustic-download-" + uuid.New().String()
+ if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {
+ return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)
+ }
+ removeScratch := func() {
+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})
+ }
+ scratchMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}
+ restoreOptions := backup.RestoreOptions{DeleteExtra: true}
+ if restoreErr := s.deps.Engine.RestoreSnapshot(ctx, dockerClient, repository, password, entry.LocalSnapshotID, scratchMount, restoreOptions); restoreErr != nil {
+ removeScratch()
+ return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreErr)
+ }
+ var removeContainer func()
+ if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, dockerClient, scratchMount); err != nil {
+ removeScratch()
+ return nil, 0, err
+ }
+ cleanup = func() {
+ removeContainer()
+ removeScratch()
+ }
+ if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); err != nil {
+ cleanup()
+ err = fmt.Errorf("failed to package Rustic snapshot for download: %w", err)
+ }
}
- dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return nil, 0, err
}
- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)
+ reader, size, err := volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)
if err != nil {
return nil, 0, err
}
- return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)
-}
-
-func (s *Service) deleteArchiveBackupInternal(ctx context.Context, entry *volume.Backup, user *usertypes.Actor) error {
- // Delete from DB first - if this fails, no changes are made. If file
- // deletion fails afterward we just have an orphan file, easier to clean up
- // than an orphan DB record pointing at a missing file.
- volumeName := entry.VolumeName
- backupID := entry.ID
- if err := s.deps.Store.Delete(ctx, backupID); err != nil {
- return err
- }
- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)
- if err != nil {
- slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backupId", backupID, "error", err.Error())
- } else {
- defer cleanup()
- filename, filenameErr := backupArchiveFilenameInternal(backupID)
- if filenameErr != nil {
- slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backupId", backupID, "error", filenameErr.Error())
- } else if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {
- slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backupId", backupID, "error", err.Error())
- }
- }
- s.logBackupDeleteEventInternal(ctx, volumeName, backupID, user)
- return nil
+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, user, database.JSON{"action": "backup_download", "backup_id": backupID, "size": size})
+ return reader, size, nil
}
// ArchiveFileTarget resolves a legacy archive backup of volumeName and the
// sanitized member path for a single-file workspace restore.
func (s *Service) ArchiveFileTarget(ctx context.Context, volumeName, backupID, rel string) (string, string, error) {
- filename, err := backupArchiveFilenameInternal(backupID)
+ filename, err := backupArchiveFilename(backupID)
if err != nil {
return "", "", common.Classify(common.ErrVolumeWorkspaceNotFound, err)
}
@@ -1011,7 +788,7 @@ func (s *Service) ArchiveFileTarget(ctx context.Context, volumeName, backupID, r
if entry.VolumeName != volumeName {
return "", "", common.Classify(common.ErrVolumeWorkspaceForbidden, errors.New("backup does not belong to volume"))
}
- cleaned, err := sanitizeBackupPathInternal(rel)
+ cleaned, err := kit.NormalizeRelativePath(strings.TrimLeft(strings.TrimSpace(rel), "/"))
if err != nil {
return "", "", common.Classify(common.ErrVolumeWorkspaceBadRequest, err)
}
@@ -1023,18 +800,6 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filena
return s.restore.RestoreArchiveMembers(ctx, containerID, filename, cleanedPaths)
}
-func (s *Service) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int, includeRemote bool) error {
- expired, err := backup.ExpiredRunIDs(ctx, s.deps.DB, "volume_backups", policyID, retentionCount)
- if err != nil || len(expired) == 0 {
- return err
- }
- entries, err := s.deps.Store.Runs(ctx, expired)
- if err != nil {
- return err
- }
- return s.deleteRusticBackupsInternal(ctx, entries, nil, includeRemote)
-}
-
// DiscoverRemoteBackups imports snapshots from every instance root on the destination; per-root failures are returned without blocking the others.
func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID string) (int, []string, error) {
if s.deps.Engine == nil || s.deps.S3Destinations == nil || s.deps.RecoveryKeys == nil {
@@ -1051,7 +816,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strin
if err != nil {
return 0, nil, errors.New("the selected S3 backup destination is not configured")
}
- roots, err := s3utils.ListRepositoryRoots(ctx, configuration, "arcane-volume-backups")
+ roots, err := s3utils.ListRepositoryRoots(ctx, configuration, backup.VolumeRoot)
if err != nil {
return 0, nil, fmt.Errorf("failed to list volume backup repositories: %w", err)
}
@@ -1070,7 +835,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strin
created := 0
var failures []string
for _, root := range roots {
- repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, root)
+ repository, repoErr := s.remoteRusticRepository(ctx, destinationID, root)
if repoErr != nil {
failures = append(failures, fmt.Sprintf("instance %s: %v", root, repoErr))
continue
@@ -1084,7 +849,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strin
if _, exists := known[snapshot.ID]; exists {
continue
}
- entry := discoveredVolumeBackupInternal(destinationID, root, snapshot)
+ entry := discoveredVolumeBackup(destinationID, root, snapshot)
if entry == nil {
continue
}
@@ -1101,8 +866,8 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strin
return created, failures, nil
}
-// discoveredVolumeBackupInternal maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.
-func discoveredVolumeBackupInternal(destinationID, root string, snapshot backup.DiscoveredSnapshot) *volume.Backup {
+// discoveredVolumeBackup maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.
+func discoveredVolumeBackup(destinationID, root string, snapshot backup.DiscoveredSnapshot) *volume.Backup {
volumeName := strings.TrimSpace(snapshot.Label)
if volumeName == "" || volumeName == systemRecoverySnapshotLabel {
return nil
@@ -1125,7 +890,7 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {
if s.deps.RecoveryKeys == nil || s.deps.Engine == nil {
return nil
}
- key, err := s.deps.RecoveryKeys.Get(ctx)
+ _, err := s.deps.RecoveryKeys.Get(ctx)
if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
return nil
}
@@ -1136,88 +901,89 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {
if err != nil {
return err
}
- local, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ local, err := s.localRusticRepository(ctx, dockerClient, true)
if err != nil {
return err
}
repositories := []backup.Repository{local}
if s.deps.S3Destinations != nil {
- destinations, listS3DestinationsByIDErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)
- if listS3DestinationsByIDErr != nil {
- return fmt.Errorf("failed to list S3 destinations for re-key: %w", listS3DestinationsByIDErr)
+ destinations, listErr := s.deps.S3Destinations.ListAllS3Destinations(ctx)
+ if listErr != nil {
+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", listErr)
}
- for destinationID := range destinations {
- remote, remoteRusticRepositoryErr := s.remoteRusticRepositoryInternal(ctx, destinationID)
+ for _, destination := range destinations {
+ remote, remoteRusticRepositoryErr := s.remoteRusticRepository(ctx, destination.ID, "")
if remoteRusticRepositoryErr != nil {
return remoteRusticRepositoryErr
}
repositories = append(repositories, remote)
}
}
- for _, repository := range repositories {
- s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)
- }
- return nil
+ _, err = s.volumeBackupPassword(ctx, dockerClient, repositories...)
+ return err
}
-// StartBackup persists a running backup and submits application-owned work.
+// StartBackup persists a running backup and submits its workflow.
func (s *Service) StartBackup(ctx context.Context, environmentID, volumeName string, user usertypes.Actor, request volume.CreateBackupRequest) (volume.BackupEntry, error) {
- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request, nil)
+ plan, err := s.resolveBackupPlan(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request, nil)
if err != nil {
return volume.BackupEntry{}, err
}
- entry, lease, err := s.prepareBackupInternal(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request.PolicyID, plan)
+ requester := backup.NewRequester(ctx, user, environmentID, authz.PermVolumesBackup)
+ if authorizeErr := s.deps.Engine.Authorize(ctx, requester); authorizeErr != nil {
+ return volume.BackupEntry{}, authorizeErr
+ }
+ entry, lease, err := s.prepareBackup(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request.PolicyID, &plan)
if err != nil {
return volume.BackupEntry{}, err
}
- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.deps.Activity, environmentID, activitytypes.TypeResourceAction,
- "volume", volumeName, volumeName, &user, "Creating backup", "Creating volume backup",
- database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID}, false)
- if activityID == "" {
- defer lease.Release(ctx)
- return volume.BackupEntry{}, s.completeBackupInternal(ctx, entry, errors.New("failed to start backup activity"))
- }
+ release := s.deps.Engine.Hold(backup.VolumeAdmissionScope, volumeName, entry.ID, lease)
+ input := manualBackupInput{Checkpoint: plan, VolumeName: volumeName, Requester: requester}
+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{
+ EnvironmentID: environmentID, ResourceType: new("volume"), ResourceID: new(volumeName), ResourceName: new(volumeName), StartedBy: &user,
+ Step: "Creating backup", LatestMessage: "Creating volume backup",
+ Metadata: database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID},
+ }, backup.AcceptedTarget(entry.ID))
+ if err != nil {
+ release(ctx)
+ return volume.BackupEntry{}, s.completeBackup(ctx, entry, err)
+ }
+ // A backup canceled before its task runs never takes the parked lease.
+ go func() {
+ background := context.WithoutCancel(ctx)
+ s.watchBackup(background, entry.ID, activityID)
+ release(background)
+ }()
entry.ActivityID = &activityID
- accepted := entryInternal(entry)
- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
- payload, err := json.Marshal(
- manualVolumeBackupInternal{
- Checkpoint: volumeBackupRecoveryInternal{
- BackupID: entry.ID,
- LocalEnabled: plan.localEnabled,
- S3Enabled: plan.s3Enabled,
- S3DestinationID: plan.s3DestinationID,
- Policy: plan.policy,
- },
- VolumeName: volumeName,
- UserID: user.ID,
- ActivityID: activityID,
- },
- )
- if err == nil {
- err = s.deps.Engine.SubmitDurableRun(
- workCtx,
- backuptypes.DurableRunCommand{
- Kind: "volume",
- RunID: entry.ID,
- ActivityID: activityID,
- Payload: payload,
- UserID: user.ID,
- EnvironmentID: environmentID,
- Permission: authz.PermVolumesBackup,
- RequestedWithKey: keyID,
- },
- lease,
- )
- }
+ return entry.Entry(), nil
+}
+
+// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.
+func (s *Service) WatchAcceptedBackups(ctx context.Context) error {
+ standalone, err := s.flow.Standalone(ctx)
if err != nil {
- lease.Release(ctx)
- err = s.completeBackupInternal(workCtx, entry, err)
- activitylib.CompleteHandlerActivity(workCtx, s.deps.Activity, activityID, "Volume backup created successfully", err)
- return volume.BackupEntry{}, err
+ return err
+ }
+ for _, run := range standalone {
+ if backupID := backup.AcceptedBackupID(run); backupID != "" && run.JobID == s.backupWorkflow.Francis().Name() {
+ go s.watchBackup(context.WithoutCancel(ctx), backupID, run.ID)
+ }
}
+ return nil
+}
- return accepted, nil
+// watchBackup fails the backup's row when its workflow ends without the task completing it, such as a cancel
+// before the task started.
+func (s *Service) watchBackup(ctx context.Context, backupID, instanceID string) {
+ outcome, err := s.flow.Wait(ctx, s.backupWorkflow, instanceID)
+ if err != nil || outcome.Status == scheduler.Succeeded {
+ return
+ }
+ if current, loadErr := s.deps.Store.Run(ctx, backupID); loadErr == nil && current.Status == backuptypes.VolumeBackupStatusRunning {
+ if failErr := s.deps.Store.Fail(ctx, backupID, cmp.Or(outcome.Message, "Backup did not run")); failErr != nil {
+ slog.WarnContext(ctx, "Failed to close a volume backup whose workflow ended early", "backupId", backupID, "error", failErr)
+ }
+ }
}
func (s *Service) ReconcileBackup(ctx context.Context, previous scheduler.Run, volumeName string) (scheduler.Outcome, error) {
@@ -1225,247 +991,189 @@ func (s *Service) ReconcileBackup(ctx context.Context, previous scheduler.Run, v
if outcome.Status == scheduler.Succeeded {
return outcome, nil
}
- for _, target := range previous.Outcome.Targets {
- if target.ID != volumeName || len(target.RecoveryData) == 0 {
- continue
- }
- var checkpoint volumeBackupRecoveryInternal
- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
- return outcome, err
- }
- entry, loadErr := s.deps.Store.Run(ctx, checkpoint.BackupID)
- if loadErr != nil {
- return outcome, loadErr
+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == volumeName && len(target.RecoveryData) > 0
+ })
+ if index < 0 {
+ return outcome, nil
+ }
+ target := previous.Outcome.Targets[index]
+ var checkpoint backupPlan
+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return outcome, err
+ }
+ entry, err := s.deps.Store.Run(ctx, checkpoint.BackupID)
+ if err != nil {
+ return outcome, err
+ }
+ // Only an interrupted backup is still running; a failed one was settled by its attempt, even if its volume
+ // was renamed since, and the caller starts a new backup under the current name.
+ if entry.Status == backuptypes.VolumeBackupStatusFailed {
+ return outcome, backup.ErrBackupSettled
+ }
+ if entry.VolumeName != volumeName {
+ return outcome, gorm.ErrRecordNotFound
+ }
+ if entry.Status != backuptypes.VolumeBackupStatusSucceeded {
+ // Resuming would copy data its stopped containers may since have changed, so the row fails and the
+ // caller starts a new backup that stops them again.
+ if checkpoint.Policy != nil && checkpoint.Policy.StopContainers {
+ if failErr := s.deps.Store.Fail(ctx, entry.ID, "Backup was interrupted while its containers were stopped"); failErr != nil {
+ return outcome, failErr
+ }
+ return outcome, backup.ErrBackupSettled
}
- if entry.VolumeName != volumeName {
- return outcome, gorm.ErrRecordNotFound
+ lease, admitted, acquireErr := s.deps.Engine.AcquireRun(ctx, backup.VolumeAdmissionScope, volumeName, "")
+ if acquireErr != nil {
+ return outcome, acquireErr
}
- if entry.Status != backuptypes.VolumeBackupStatusSucceeded {
- if checkpoint.Policy != nil && checkpoint.Policy.StopContainers {
- return outcome, nil
- }
- lease, admitted, err := s.deps.Engine.AcquireDurableRun(ctx, previous.ID, backup.VolumeAdmissionScope, volumeName)
- if err != nil {
- return outcome, err
- }
- if !admitted {
- return outcome, nil
- }
- defer lease.Release(ctx) //nolint:gocritic // The matching recovery target always returns before the loop advances.
- if resumeBackupErr := s.resumeBackupInternal(ctx, previous, entry, checkpoint); resumeBackupErr != nil {
- return outcome, resumeBackupErr
- }
+ // Another backup of this volume is running; the task is handed back, without spending its one attempt,
+ // so a later delivery resumes this one rather than leaving it running.
+ if !admitted {
+ return outcome, fmt.Errorf("%w: %w", actor.ErrJobRejected, s.deps.AlreadyRunning)
}
- target.Status = scheduler.Succeeded
- if err := jobcontext.Progress(ctx, target); err != nil {
- return outcome, err
+ defer lease.Release(ctx)
+ if resumeErr := s.resumeBackup(ctx, previous, entry, checkpoint); resumeErr != nil {
+ return outcome, resumeErr
}
- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
}
- return outcome, nil
+ target.Status = scheduler.Succeeded
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return outcome, progressErr
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
}
-func (
- s *Service,
-) recoverSnapshotInternal(
- ctx context.Context,
- dockerClient *client.Client,
- repository backup.Repository,
- entry *volume.Backup,
- snapshotID string,
-) (
- backup.Snapshot,
- bool,
- error,
-) {
- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+// resumeBackup adopts snapshots an interrupted run already wrote, then finishes the backup.
+func (s *Service) resumeBackup(ctx context.Context, previous scheduler.Run, entry *volume.Backup, checkpoint backupPlan) error {
+ dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
- return backup.Snapshot{}, false, err
+ return err
}
- return s.deps.Engine.FindRunSnapshot(ctx, dockerClient, repository, password, entry.ID, snapshotID)
-}
-
-func (
- s *Service,
-) observeBackupDestinationsInternal(
- ctx context.Context,
- dockerClient *client.Client,
- previous scheduler.Run,
- entry *volume.Backup,
- checkpoint volumeBackupRecoveryInternal,
-) (
- backup.Repository,
- backup.Repository,
- error,
-) {
var local, remote backup.Repository
- var err error
+ var repositories []backup.Repository
if checkpoint.LocalEnabled {
- local, err = s.localRusticRepositoryInternal(ctx, dockerClient, false)
- if err != nil {
- return local, remote, err
- }
- if entry.LocalSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "local") {
- snapshot, found, recoverSnapshotErr := s.recoverSnapshotInternal(ctx, dockerClient, local, entry, entry.LocalSnapshotID)
- if recoverSnapshotErr != nil {
- return local, remote, recoverSnapshotErr
- }
- if found {
- entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size
- }
+ if local, err = s.localRusticRepository(ctx, dockerClient, false); err != nil {
+ return err
}
+ repositories = append(repositories, local)
}
if checkpoint.S3Enabled {
- remote, err = s.remoteRusticRepositoryInternal(ctx, checkpoint.S3DestinationID)
- if err != nil {
- return local, remote, err
- }
- if entry.RemoteSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "remote") {
- snapshot, found, recoverSnapshotErr2 := s.recoverSnapshotInternal(ctx, dockerClient, remote, entry, entry.RemoteSnapshotID)
- if recoverSnapshotErr2 != nil {
- return local, remote, recoverSnapshotErr2
- }
- if found {
- entry.RemoteSnapshotID = snapshot.ID
- if entry.Size == 0 {
- entry.Size = snapshot.Size
- }
- }
+ if remote, err = s.remoteRusticRepository(ctx, checkpoint.S3DestinationID, ""); err != nil {
+ return err
}
+ repositories = append(repositories, remote)
}
- return local, remote, nil
-}
-
-func (s *Service) resumeBackupInternal(ctx context.Context, previous scheduler.Run, entry *volume.Backup, checkpoint volumeBackupRecoveryInternal) error {
- dockerClient, err := s.deps.Docker.GetClient(ctx)
+ password, err := s.volumeBackupPassword(ctx, dockerClient, repositories...)
if err != nil {
return err
}
- local, remote, err := s.observeBackupDestinationsInternal(ctx, dockerClient, previous, entry, checkpoint)
- if err != nil {
- return err
+ attempted := func(destination string) bool {
+ return slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == entry.ID+":"+destination })
}
- if checkpoint.LocalEnabled && entry.LocalSnapshotID == "" && entry.RemoteSnapshotID != "" {
- password, volumeBackupPasswordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remote)
- if volumeBackupPasswordErr != nil {
- return volumeBackupPasswordErr
+ if checkpoint.LocalEnabled && (entry.LocalSnapshotID != "" || attempted("local")) {
+ snapshot, found, findErr := s.deps.Engine.FindRunSnapshot(ctx, dockerClient, local, password, entry.ID, entry.LocalSnapshotID)
+ if findErr != nil {
+ return findErr
}
- snapshot, volumeBackupPasswordErr := s.deps.Engine.Replicate(ctx, dockerClient, remote, entry.RemoteSnapshotID, local, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
- if volumeBackupPasswordErr != nil {
- return volumeBackupPasswordErr
+ if found {
+ entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size
}
- entry.LocalSnapshotID = snapshot.ID
}
- if operationErr := s.deps.Store.Save(ctx, entry); operationErr != nil {
- return operationErr
+ if checkpoint.S3Enabled && (entry.RemoteSnapshotID != "" || attempted("remote")) {
+ snapshot, found, findErr := s.deps.Engine.FindRunSnapshot(ctx, dockerClient, remote, password, entry.ID, entry.RemoteSnapshotID)
+ if findErr != nil {
+ return findErr
+ }
+ if found {
+ entry.RemoteSnapshotID, entry.Size = snapshot.ID, cmp.Or(entry.Size, snapshot.Size)
+ }
+ }
+ if checkpoint.LocalEnabled && entry.LocalSnapshotID == "" && entry.RemoteSnapshotID != "" {
+ snapshot, replicateErr := s.deps.Engine.Replicate(ctx, dockerClient, remote, entry.RemoteSnapshotID, local, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
+ if replicateErr != nil {
+ return replicateErr
+ }
+ entry.LocalSnapshotID = snapshot.ID
}
- plan := backupPlanInternal{
- localEnabled: checkpoint.LocalEnabled,
- s3Enabled: checkpoint.S3Enabled,
- s3DestinationID: checkpoint.S3DestinationID,
- policy: checkpoint.Policy,
- destination: entry.Destination,
+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {
+ return saveErr
}
- err = s.executeBackupInternal(ctx, entry, usertypes.SystemUser, plan)
- return s.completeBackupInternal(ctx, entry, err)
-}
-
-type manualVolumeBackupInternal struct {
- Checkpoint volumeBackupRecoveryInternal `json:"checkpoint"`
- VolumeName string `json:"volumeName"`
- UserID string `json:"userId"`
- ActivityID string `json:"activityId"`
+ return s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, usertypes.SystemUser, checkpoint))
}
-func (s *Service) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
- var command manualVolumeBackupInternal
- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {
- return decodeErr
+// runManualBackup is the volume-backup step. A delivery that finds the checkpoint it recorded
+// resumes that backup; otherwise it takes over the admission StartBackup won and runs it.
+func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, err error) {
+ var input manualBackupInput
+ if decodeErr := t.Payload(&input); decodeErr != nil {
+ return nil, decodeErr
}
defer func() {
- if ctx.Err() == nil {
- if err != nil {
- saveErr := s.deps.Store.Fail(ctx, command.Checkpoint.BackupID, err.Error())
- err = errors.Join(err, saveErr)
- }
- activitylib.CompleteHandlerActivity(ctx, s.deps.Activity, command.ActivityID, "Volume backup created successfully", err)
+ if err != nil && ctx.Err() == nil && !errors.Is(err, actor.ErrJobRejected) && !errors.Is(err, backup.ErrBackupSettled) {
+ err = errors.Join(err, s.deps.Store.Fail(ctx, input.Checkpoint.BackupID, err.Error()))
}
}()
- entry, loadBackupErr := s.deps.Store.Run(ctx, command.Checkpoint.BackupID)
- if loadBackupErr != nil {
- return loadBackupErr
+ previous, _ := jobcontext.Run(ctx)
+ resuming := slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == input.VolumeName && len(target.RecoveryData) > 0
+ })
+ if !resuming {
+ lease, admitted, acquireErr := s.deps.Engine.AcquireRun(ctx, backup.VolumeAdmissionScope, input.VolumeName, input.Checkpoint.BackupID)
+ if acquireErr != nil {
+ return nil, acquireErr
+ }
+ if !admitted {
+ return nil, s.deps.AlreadyRunning
+ }
+ defer lease.Release(ctx)
}
-
- if entry.Status == backuptypes.VolumeBackupStatusSucceeded {
- return nil
+ if authorizeErr := s.deps.Engine.Authorize(ctx, input.Requester); authorizeErr != nil {
+ return nil, authorizeErr
}
- if interrupted {
- previous, _ := jobcontext.Run(ctx)
- outcome, recoveryErr := s.ReconcileBackup(ctx, previous, command.VolumeName)
- if recoveryErr != nil {
- return recoveryErr
+ succeeded := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Volume backup created successfully"}
+ if resuming {
+ outcome, reconcileErr := s.ReconcileBackup(ctx, previous, input.VolumeName)
+ // A settled row already records why it failed, so the run reports that rather than the sentinel.
+ if errors.Is(reconcileErr, backup.ErrBackupSettled) {
+ if entry, loadErr := s.deps.Store.Run(ctx, input.Checkpoint.BackupID); loadErr == nil && entry.Error != "" {
+ return nil, fmt.Errorf("%w: %s", backup.ErrBackupSettled, entry.Error)
+ }
+ }
+ if reconcileErr != nil {
+ return nil, reconcileErr
}
if outcome.Status != scheduler.Succeeded {
- return errors.New(outcome.Message)
+ return nil, errors.New(outcome.Message)
}
- return nil
+ return succeeded, nil
}
- lease, admitted, err := s.deps.Engine.AcquireDurableRun(ctx, runID, backup.VolumeAdmissionScope, command.VolumeName)
+ entry, err := s.deps.Store.Run(ctx, input.Checkpoint.BackupID)
if err != nil {
- return err
- }
- if !admitted {
- return s.deps.AlreadyRunning
+ return nil, err
}
- defer lease.Release(ctx)
- actor := usertypes.SystemUser
- if command.UserID != "" && command.UserID != "agent" {
+ requestedBy := usertypes.SystemUser
+ if input.Requester.UserID != "" && input.Requester.UserID != "agent" {
var localUser userdomain.User
- if loadUserErr := s.deps.DB.WithContext(ctx).Where("id = ?", command.UserID).First(&localUser).Error; loadUserErr != nil {
- return loadUserErr
+ if loadUserErr := s.deps.DB.WithContext(ctx).Where("id = ?", input.Requester.UserID).First(&localUser).Error; loadUserErr != nil {
+ return nil, loadUserErr
}
- actor = *localUser.Actor()
+ requestedBy = *localUser.Actor()
}
- checkpoint, err := json.Marshal(command.Checkpoint)
+ checkpoint, err := json.Marshal(input.Checkpoint)
if err != nil {
- return err
- }
- if progressErr := jobcontext.Progress(
- ctx,
- scheduler.TargetOutcome{
- ResourceType: "volume_backup",
- ID: command.VolumeName,
- Status: scheduler.Running,
- RecoveryData: checkpoint,
- ActivityID: command.ActivityID,
- },
- ); progressErr != nil {
- return progressErr
+ return nil, err
}
- plan := backupPlanInternal{
- localEnabled: command.Checkpoint.LocalEnabled,
- s3Enabled: command.Checkpoint.S3Enabled,
- s3DestinationID: command.Checkpoint.S3DestinationID,
- policy: command.Checkpoint.Policy,
- destination: entry.Destination,
+ target := scheduler.TargetOutcome{ResourceType: "volume_backup", ID: input.VolumeName, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return nil, progressErr
}
- err = s.executeBackupInternal(ctx, entry, actor, plan)
+ err = s.executeBackup(ctx, entry, requestedBy, input.Checkpoint)
if ctx.Err() != nil {
- return ctx.Err()
- }
- return s.completeBackupInternal(ctx, entry, err)
-}
-
-func (s *Service) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {
- var command manualVolumeBackupInternal
- if err := json.Unmarshal(payload, &command); err != nil {
- return err
+ return nil, ctx.Err()
}
- entry, err := s.deps.Store.Run(ctx, command.Checkpoint.BackupID)
- if err != nil {
- return err
+ if completeErr := s.completeBackup(ctx, entry, err); completeErr != nil {
+ return nil, completeErr
}
- err = s.deps.Store.Fail(ctx, entry.ID, runErr.Error())
- activitylib.CompleteHandlerActivity(ctx, s.deps.Activity, command.ActivityID, "Volume backup created successfully", runErr)
- return err
+ return succeeded, nil
}
diff --git a/backend/internal/volume/children/backup/service_test.go b/backend/internal/volume/children/backup/service_test.go
index 530c2afbc..3c70af66f 100644
--- a/backend/internal/volume/children/backup/service_test.go
+++ b/backend/internal/volume/children/backup/service_test.go
@@ -16,7 +16,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
@@ -24,11 +24,11 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/backup/children/policies"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {
+func applyVolumeBackupMigrations(t *testing.T, gormDB *gorm.DB) {
t.Helper()
for _, name := range []string{"032_add_volume_backups.sql", "073_add_backup_support.sql", "087_add_volume_backup_remote_instance.sql"} {
migration, err := os.ReadFile("../../../../resources/migrations/sqlite/" + name)
@@ -37,41 +37,32 @@ func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {
}
}
-func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {
- t.Helper()
- runtime := francistest.New(t)
- gate := runs.NewAdmission(runtime.Service(), t.Name())
- require.NoError(t, gate.Register(runtime))
- francistest.Start(t, runtime)
- return gate
-}
-
-type volumeBackupPolicySchedulerInternal struct {
+type volumeBackupPolicyScheduler struct {
jobs map[string]scheduler.Job
}
-func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {
+func (s *volumeBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {
s.jobs[job.Name()] = job
return nil
}
-func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {
+func (s *volumeBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {
delete(s.jobs, name)
}
-func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {
+func (s *volumeBackupPolicyScheduler) HasJob(name string) bool {
_, ok := s.jobs[name]
return ok
}
-func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
+func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {
return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil
}
func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- applyVolumeBackupMigrationsInternal(t, gormDB)
+ applyVolumeBackupMigrations(t, gormDB)
policyID := "policy-1"
require.NoError(t, gormDB.Exec(
@@ -83,20 +74,15 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
"failed-run", "app-data", 0, time.Now(), policyID, backuptypes.VolumeBackupStatusFailed,
).Error)
- service := NewService(Dependencies{DB: &database.DB{DB: gormDB}})
- require.NoError(t, service.applyVolumeBackupRetentionInternal(t.Context(), policyID, 1, true))
-
- var backups []volume.Backup
- require.NoError(t, gormDB.Table("volume_backups").Order("created_at ASC").Find(&backups).Error)
- require.Len(t, backups, 2)
- require.Equal(t, "snapshot-1", backups[0].LocalSnapshotID)
- require.Equal(t, backuptypes.VolumeBackupStatusFailed, backups[1].Status)
+ expired, err := backup.ExpiredRunIDs(t.Context(), &database.DB{DB: gormDB}, "volume_backups", policyID, 1)
+ require.NoError(t, err)
+ require.Empty(t, expired)
}
func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-scheduled-activity?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&policies.VolumeBackupPolicy{}, &activity.Activity{}))
+ require.NoError(t, gormDB.AutoMigrate(&policies.VolumeBackupPolicy{}, &activity.Activity{}, &activity.ActivityMessage{}))
db := &database.DB{DB: gormDB}
policy := &policies.VolumeBackupPolicy{
VolumeName: "app-data",
@@ -106,23 +92,21 @@ func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
LocalEnabled: true,
}
require.NoError(t, gormDB.Create(policy).Error)
- gate := newVolumeAdmissionForTestInternal(t)
- engine := backup.NewEngine(t.Context(), gate, nil)
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ gate := runs.NewAdmission(harness.Runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(harness.Runtime))
+ engine := backup.NewEngine(gate, nil)
t.Cleanup(func() { require.NoError(t, engine.Stop(context.WithoutCancel(t.Context()))) })
- errAlreadyRunning := errors.New("a backup is already running for this volume")
- service := NewService(Dependencies{
- DB: db,
- Activity: activity.NewActivityService(db, nil),
- Engine: engine,
- AlreadyRunning: errAlreadyRunning,
- })
- jobScheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}
+ service := NewService(Dependencies{DB: db, Engine: engine, AlreadyRunning: errors.New("a backup is already running for this volume")})
+ require.NoError(t, service.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
+ jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}
require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))
service.RegisterJobsOnStartup(t.Context())
job, registered := jobScheduler.jobs["volume-backup:"+policy.ID]
require.True(t, registered)
- // Holding the volume's admission lease makes the scheduled run fail with
- // "already running", which still must record a failed activity.
+ // Holding the volume's admission lease skips the scheduled run, which still completes its activity.
lease, admitted, err := engine.TryAcquireRun(t.Context(), backup.VolumeAdmissionScope, policy.VolumeName)
require.NoError(t, err)
require.True(t, admitted)
@@ -134,7 +118,7 @@ func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
var backupActivity activity.Activity
require.NoError(t, gormDB.Where("resource_type = ?", "volume_backup").First(&backupActivity).Error)
- require.Equal(t, activitytypes.StatusFailed, backupActivity.Status)
+ require.Equal(t, activitytypes.StatusSuccess, backupActivity.Status)
require.Equal(t, "scheduled_volume_backup", backupActivity.Metadata["action"])
require.Equal(t, policy.Schedule, backupActivity.Metadata["schedule"])
}
@@ -171,13 +155,13 @@ func TestVolumeBackupPasswordPrefersRecoveryKey(t *testing.T) {
RecoveryKeys: recoveryKeys,
})
- password, err := service.volumeBackupPasswordInternal(t.Context(), nil)
+ password, err := service.volumeBackupPassword(t.Context(), nil)
require.NoError(t, err)
- require.Equal(t, kit.SHA256Hex(legacyVolumePasswordSaltInternal+service.deps.EncryptionKey), password)
+ require.Equal(t, kit.SHA256Hex(legacyVolumePasswordSalt+service.deps.EncryptionKey), password)
recoveryKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"
require.NoError(t, recoveryKeys.Set(t.Context(), recoveryKey))
- password, err = service.volumeBackupPasswordInternal(t.Context(), nil)
+ password, err = service.volumeBackupPassword(t.Context(), nil)
require.NoError(t, err)
require.Equal(t, recoveryKey, password)
}
@@ -186,7 +170,7 @@ func TestDiscoveredVolumeBackupMapping(t *testing.T) {
snapshot := backup.DiscoveredSnapshot{ID: "abc123", Label: "app-data", Time: time.Unix(1700000000, 0).UTC()}
snapshot.Summary.TotalBytesProcessed = 2048
- entry := discoveredVolumeBackupInternal("destination-1", "instance-a", snapshot)
+ entry := discoveredVolumeBackup("destination-1", "instance-a", snapshot)
require.NotNil(t, entry)
require.Equal(t, "app-data", entry.VolumeName)
require.Equal(t, "remote-destination-1-instance-a-abc123", entry.ID)
@@ -201,11 +185,11 @@ func TestDiscoveredVolumeBackupMapping(t *testing.T) {
// Snapshots without a volume label, and system recovery snapshots, never
// map to a volume backup.
- require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))
- require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))
+ require.Nil(t, discoveredVolumeBackup("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))
+ require.Nil(t, discoveredVolumeBackup("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))
// A missing snapshot timestamp falls back to now.
- fallback := discoveredVolumeBackupInternal("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})
+ fallback := discoveredVolumeBackup("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})
require.NotNil(t, fallback)
require.False(t, fallback.CreatedAt.IsZero())
}
diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.go
index b6ec321d9..39eb24532 100644
--- a/backend/internal/volume/handler.go
+++ b/backend/internal/volume/handler.go
@@ -307,15 +307,7 @@ func (h *VolumeHandler) RemoveVolume(ctx context.Context, input *RemoveVolumeInp
return nil, huma.Error500InternalServerError("Failed to delete volume: " + err.Error())
}
- return &handlerutil.Out[base.MessageResponse]{
- Body: base.ApiResponse[base.MessageResponse]{
- Success: true,
- Data: base.MessageResponse{
- Message: "Volume removed successfully",
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- },
- },
- }, nil
+ return handlerutil.MessageOutput("Volume removed successfully", activityID), nil
}
// PruneVolumes removes all unused Docker volumes.
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index bc41c12e4..731ad0741 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -22,11 +22,10 @@ import (
"github.com/moby/moby/client"
"github.com/samber/mo"
"go.getarcane.app/docker"
- "go.getarcane.app/kit/pkg"
+ kit "go.getarcane.app/kit/pkg"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -43,6 +42,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumes"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"
@@ -75,7 +75,6 @@ func NewVolumeService(
db *database.DB,
dockerService *dockerInternal.DockerClientService,
eventService *event.EventService,
- activityService *activity.ActivityService,
settingsService *settings.SettingsService,
containerService *containerdomain.ContainerService,
imageService *image.ImageService,
@@ -113,7 +112,6 @@ func NewVolumeService(
DB: db,
Docker: dockerService,
Events: eventService,
- Activity: activityService,
Settings: settingsService,
Engine: engine,
S3Destinations: s3Destinations,
@@ -194,12 +192,22 @@ func (s *VolumeService) ReconcileBackup(ctx context.Context, previous scheduler.
}
// ReconcileInterruptedBackups runs before this process can accept backup work.
+// RegisterWorkflows defines the backup workflows while the host is still unstarted.
+func (s *VolumeService) RegisterWorkflows(engine *flow.Engine) error {
+ return s.backup.RegisterWorkflows(engine)
+}
+
+// ReconcileInterruptedBackups fails backups left running by the previous process and watches the manual
+// backups that saved workflows resume.
func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning)
if len(protectedIDs) > 0 {
query = query.Where("id NOT IN ?", protectedIDs)
}
- return query.Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error
+ if err := query.Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error; err != nil {
+ return err
+ }
+ return s.backup.WatchAcceptedBackups(ctx)
}
// PruneLocalBackupRepository frees the space of deleted local volume backups.
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 2e0248f52..63d88ab34 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -124,7 +124,7 @@ func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {
require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (id, volume_name, schedule) VALUES (?, ?, ?)", "policy-1", "source-data", "0 0 2 * * *").Error)
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "source-data", PolicyID: "policy-1"}).Error)
- service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, service.renameVolumeMetadataInternal(t.Context(), "source-data", "renamed-data"))
var renamedPolicy string
@@ -271,7 +271,7 @@ func TestVolumeWorkspaceReadsWaitForMutationLock(t *testing.T) {
t.Cleanup(server.Close)
dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))
- service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)
unlockMutation := service.workspaceLocks.Lock("workspace-volume")
readStarted := make(chan struct{})
readDone := make(chan error, 1)
@@ -351,7 +351,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {
t.Cleanup(server.Close)
dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))
- service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)
service.helperByVolume["workspace-volume"] = &volumeHelper{id: "helper", lastUsedAt: time.Now(), protocol: types.ProtocolVersion}
reader, size, err := service.workspace.DownloadVolumeWorkspaceFile(t.Context(), "workspace-volume", "file.txt")
require.NoError(t, err)
@@ -417,7 +417,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {
t.Cleanup(server.Close)
dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)
- service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil)
uploadIndex := 0
workspace, err := service.workspace.UpdateVolumeWorkspace(t.Context(), "workspace-volume", volumetypes.WorkspaceUpdateManifest{
FileTreeRevision: "stale",
@@ -457,7 +457,7 @@ func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {
S3DestinationID: destination.ID,
}).Error)
- service := NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, s3.NewS3DestinationService(db, nil), nil, nil)
+ service := NewVolumeService(db, nil, nil, nil, nil, nil, nil, s3.NewS3DestinationService(db, nil), nil, nil)
backups, _, err := service.backup.List(t.Context(), "app-data", pagination.QueryParams{})
require.NoError(t, err)
require.Len(t, backups, 1)
@@ -472,7 +472,7 @@ func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicyFromStore(t *testing.T
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "first-policy", Status: VolumeBackupStatusSucceeded}).Error)
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "second-policy", Status: VolumeBackupStatusFailed}).Error)
- store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()
+ store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()
first, err := store.Latest(t.Context(), "first-policy")
require.NoError(t, err)
require.Equal(t, "succeeded", first.Status)
diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.go
index 6436e27a2..8c169bfd2 100644
--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go
+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go
@@ -429,6 +429,10 @@ func (
return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
}
+ // The name can move to another image after it was selected; never store that image's findings under imageID.
+ if scanned := trivyReport.Metadata.ImageID; scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {
+ return nil, fmt.Errorf("%s now refers to image %s, not the selected image %s", imageName, scanned, imageID)
+ }
result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), 0)
result.ScannerVersion = scannerVersion
result.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, stdoutFile)
diff --git a/backend/internal/vulnerability/children/scan/service.go b/backend/internal/vulnerability/children/scan/service.go
index 11bbe7194..dda53a942 100644
--- a/backend/internal/vulnerability/children/scan/service.go
+++ b/backend/internal/vulnerability/children/scan/service.go
@@ -6,20 +6,19 @@ import (
"errors"
"fmt"
"log/slog"
+ "slices"
"strings"
"sync"
- "sync/atomic"
"time"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/features"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/vulnerability"
- "github.com/moby/moby/api/types/image"
"github.com/moby/moby/api/types/registry"
+ "github.com/moby/moby/client"
"github.com/samber/mo"
- "go.getarcane.app/kit/pkg"
- "golang.org/x/sync/errgroup"
+ kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -32,14 +31,23 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan/children/trivy"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
const (
- scansTableInternal = "vulnerability_scans"
- scanStaleTimeout = 30 * time.Minute
+ scansTable = "vulnerability_scans"
+ scanStaleTimeout = 30 * time.Minute
+ scheduledScanMaxAge = 7 * 24 * time.Hour
+ // scanPoolIdle removes shared scan containers left behind by a scan that never finalized.
+ scanPoolIdle = 2 * time.Minute
+ // scanPoolSetupTimeout bounds creating the shared scan containers, including pulling the scanner image.
+ scanPoolSetupTimeout = 10 * time.Minute
)
+// scheduledScanUser is the actor recorded for scheduled scans.
+var scheduledScanUser = usertypes.Actor{Username: "System"}
+
// Records updates scan records, whose model the vulnerability domain owns.
type Records struct {
MarkScanning func(ctx context.Context, imageID string) error
@@ -78,10 +86,39 @@ type Service struct {
saveResult func(ctx context.Context, result *vulnerability.ScanResult) error
ensureSummary func(result *vulnerability.ScanResult)
filterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)
- // scanPhases tracks live scan phases in-memory for in-progress scans.
- scanPhases sync.Map // map[string]vulnerability.ScanPhase
+ scanPhases sync.Map // map[string]vulnerability.ScanPhase
+ poolMu sync.Mutex
+ pool *scanPool
}
+type (
+ // Target is one image a scheduled scan covers.
+ Target struct {
+ ImageID string `json:"imageId"`
+ ImageName string `json:"imageName"`
+ ScannerVersion string `json:"scannerVersion,omitempty"`
+ }
+
+ // Fixable summarizes one image's fixable findings for the scan notification.
+ Fixable struct {
+ Total int `json:"total,omitempty"`
+ Severity map[string]int `json:"severity,omitempty"`
+ CVEs []string `json:"cves,omitempty"`
+ }
+
+ // scanPool is the set of long-running Trivy containers scheduled scans exec
+ // into. Scan tasks share it across deliveries until finalize or idleness removes it.
+ scanPool struct {
+ containers chan string
+ files map[string][]byte
+ cleanup func()
+ users int
+ idle *time.Timer
+ // retired closes when a container could not be stopped; waiting scans then retry on a fresh pool.
+ retired chan struct{}
+ }
+)
+
func NewService(deps Dependencies) *Service {
s := &Service{
db: deps.DB,
@@ -95,12 +132,12 @@ func NewService(deps Dependencies) *Service {
ensureSummary: deps.EnsureSummary,
filterIgnored: deps.FilterIgnored,
}
- s.trivy = trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, s.setScanPhaseInternal)
+ s.trivy = trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, s.setScanPhase)
return s
}
-// scanStateRowInternal is the freshness projection of a scan record.
-type scanStateRowInternal struct {
+// scanStateRow is the freshness projection of a scan record.
+type scanStateRow struct {
ID string `gorm:"column:id"`
ImageName string `gorm:"column:image_name"`
Status string `gorm:"column:status"`
@@ -109,150 +146,149 @@ type scanStateRowInternal struct {
ScanPolicy string `gorm:"column:scan_policy"`
}
-func (s *Service) setScanPhaseInternal(imageID string, phase vulnerability.ScanPhase) {
+func (s *Service) setScanPhase(imageID string, phase vulnerability.ScanPhase) {
if imageID == "" || phase == "" {
return
}
s.scanPhases.Store(imageID, phase)
}
-func (s *Service) clearScanPhaseInternal(imageID string) {
- if imageID == "" {
- return
- }
- s.scanPhases.Delete(imageID)
-}
-
-func (s *Service) getScanPhaseInternal(imageID string) vulnerability.ScanPhase {
- if imageID == "" {
- return ""
- }
-
- phase, ok := s.scanPhases.Load(imageID)
- if !ok {
- return ""
- }
-
- if typed, localOk := phase.(vulnerability.ScanPhase); localOk {
- return typed
- }
-
- if str, localOk2 := phase.(string); localOk2 {
- return vulnerability.ScanPhase(str)
- }
-
- return ""
-}
-
-func (s *Service) resolveScanPhaseInternal(imageID string, status vulnerability.ScanStatus) vulnerability.ScanPhase {
+// resolveScanPhase reports the live phase of an in-progress scan, falling back to one implied by its status.
+func (s *Service) resolveScanPhase(imageID string, status vulnerability.ScanStatus) vulnerability.ScanPhase {
if status != vulnerability.ScanStatusPending && status != vulnerability.ScanStatusScanning {
return ""
}
-
- if phase := s.getScanPhaseInternal(imageID); phase != "" {
- return phase
- }
-
- if status == vulnerability.ScanStatusPending {
- return vulnerability.ScanPhaseCreatingContainer
- }
- return vulnerability.ScanPhaseScanningImage
-}
-
-func (s *Service) applyScanPhaseToSummaryInternal(summary *vulnerability.ScanSummary) {
- if summary == nil {
- return
- }
-
- summary.ScanPhase = s.resolveScanPhaseInternal(summary.ImageID, summary.Status)
-}
-
-func (s *Service) applyScanPhaseToResultInternal(result *vulnerability.ScanResult) {
- if result == nil {
- return
+ if phase, ok := s.scanPhases.Load(imageID); ok {
+ if typed, isPhase := phase.(vulnerability.ScanPhase); isPhase && typed != "" {
+ return typed
+ }
}
-
- result.ScanPhase = s.resolveScanPhaseInternal(result.ImageID, result.Status)
+ return kit.Ternary(status == vulnerability.ScanStatusPending, vulnerability.ScanPhaseCreatingContainer, vulnerability.ScanPhaseScanningImage)
}
-// ScanImage scans an image for vulnerabilities using Trivy
+// ScanImage queues a Trivy scan of an image and runs it in the background.
func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user usertypes.Actor) (*vulnerability.ScanResult, error) {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
return nil, err
}
scanCtx := utils.ActivityRuntimeContext(ctx, nil)
- // Get Docker client to inspect the image
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
-
- // Inspect the image to get details
imageInspect, err := dockerClient.ImageInspect(scanCtx, imageID)
if err != nil {
return nil, fmt.Errorf("failed to inspect image: %w", err)
}
-
- // Determine image name to scan
- imageName := imageID
- if len(imageInspect.RepoTags) > 0 {
- imageName = imageInspect.RepoTags[0]
- } else if len(imageInspect.RepoDigests) > 0 {
- imageName = imageInspect.RepoDigests[0]
- }
+ imageName := slices.Concat(imageInspect.RepoTags, imageInspect.RepoDigests, []string{imageID})[0]
slotID, releaseSlot, err := s.trivy.AcquireSlot(scanCtx, imageID, false)
if err != nil {
return nil, err
}
status := kit.Ternary(releaseSlot != nil, vulnerability.ScanStatusScanning, vulnerability.ScanStatusPending)
-
- slog.InfoContext(scanCtx,
- "vulnerability scan queued",
- "environmentId", envID,
- "imageId", imageID,
- "imageName", imageName,
- )
- activityID := s.startVulnerabilityScanActivityInternal(scanCtx, envID, imageID, imageName, &user)
- // Make the scan cancelable: cancelling the activity cancels scanCtx, which
- // unblocks the Trivy container wait and triggers its (cancel-safe) teardown.
+ slog.InfoContext(scanCtx, "vulnerability scan queued", "environmentId", envID, "imageId", imageID, "imageName", imageName)
+
+ activityID := ""
+ if s.activityService != nil {
+ scanActivity, startErr := s.activityService.StartActivity(scanCtx, activity.StartActivityRequest{
+ EnvironmentID: envID,
+ Type: activitytypes.TypeVulnerabilityScan,
+ ResourceType: new("image"),
+ ResourceID: &imageID,
+ ResourceName: &imageName,
+ StartedBy: &user,
+ Progress: new(0),
+ Step: string(vulnerability.ScanPhaseCreatingContainer),
+ LatestMessage: "Vulnerability scan queued",
+ })
+ if startErr != nil {
+ slog.WarnContext(scanCtx, "failed to create vulnerability scan activity", "error", startErr, "imageId", imageID)
+ } else {
+ activityID = scanActivity.ID
+ }
+ }
+ activityRef := mo.EmptyableToOption(activityID).ToPointer()
+ // Cancelling the activity cancels scanCtx, which stops the Trivy container wait and tears it down.
scanCtx = s.activityService.Track(scanCtx, activityID)
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseCreatingContainer)
+ s.setScanPhase(imageID, vulnerability.ScanPhaseCreatingContainer)
- // Create pending scan record
- pendingResult := &vulnerability.ScanResult{
- ImageID: imageID,
- ImageName: imageName,
- ScanTime: time.Now(),
- Status: status,
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- }
- s.applyScanPhaseToResultInternal(pendingResult)
+ pendingResult := &vulnerability.ScanResult{ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: status, ActivityID: activityRef}
+ pendingResult.ScanPhase = s.resolveScanPhase(imageID, status)
if saveErr := s.saveResult(scanCtx, pendingResult); saveErr != nil {
- slog.WarnContext(scanCtx,
- "failed to save pending scan result",
- "error", saveErr,
- "imageId", imageID,
- "imageName", imageName,
- )
- } else {
- slog.DebugContext(scanCtx,
- "saved pending vulnerability scan record",
- "imageId", imageID,
- "imageName", imageName,
- "scanTime", pendingResult.ScanTime,
- )
- }
-
- go s.scanImageInBackgroundInternal(scanCtx, envID, imageID, imageName, user, activityID, slotID, releaseSlot)
+ slog.WarnContext(scanCtx, "failed to save pending scan result", "error", saveErr, "imageId", imageID, "imageName", imageName)
+ }
+
+ go func() {
+ var runErr error
+ if releaseSlot == nil {
+ slotID, releaseSlot, runErr = s.trivy.AcquireSlot(scanCtx, imageID, true)
+ }
+ if releaseSlot != nil {
+ defer releaseSlot()
+ if updateErr := s.records.MarkScanning(scanCtx, imageID); updateErr != nil {
+ slog.WarnContext(scanCtx, "failed to mark queued scan as running", "imageId", imageID, "error", updateErr)
+ }
+ }
+ defer s.scanPhases.Delete(imageID)
+ // fail records a failed scan; a scan that never started carries no duration.
+ fail := func(message string, duration int64) {
+ s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)
+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing failed scan result")
+ failed := &vulnerability.ScanResult{
+ ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: vulnerability.ScanStatusFailed,
+ ActivityID: activityRef, Error: message, Duration: duration,
+ }
+ if saveErr := s.saveResult(scanCtx, failed); saveErr != nil {
+ slog.WarnContext(scanCtx, "failed to save failed scan result", "error", saveErr)
+ }
+ slog.WarnContext(scanCtx, "background vulnerability scan failed", "environmentId", envID, "imageId", imageID, "imageName", imageName, "error", message)
+ s.completeVulnerabilityScanActivity(scanCtx, activityID, false, message)
+ }
+
+ s.setScanPhase(imageID, vulnerability.ScanPhaseCreatingContainer)
+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseCreatingContainer, 10, "Preparing vulnerability scanner")
+ var scannerImage string
+ if runErr == nil {
+ scannerImage, runErr = s.trivy.EnsureImage(scanCtx)
+ }
+ if runErr != nil {
+ fail(kit.Ternary(errors.Is(runErr, common.ErrFeatureDisabled), runErr.Error(), "Trivy scanner is not available: "+runErr.Error()), 0)
+ return
+ }
+
+ s.setScanPhase(imageID, vulnerability.ScanPhaseScanningImage)
+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseScanningImage, 45, "Scanning image for vulnerabilities")
+ startTime := time.Now()
+ result, runErr := s.trivy.Scan(scanCtx, scannerImage, imageName, imageID, slotID)
+ duration := time.Since(startTime).Milliseconds()
+ if runErr != nil {
+ s.logScanEvent(scanCtx, event.EventTypeImageScan, "vulnerability_scan", envID, imageID, imageName, user, runErr.Error())
+ fail(runErr.Error(), duration)
+ return
+ }
+
+ result.Duration = duration
+ result.ActivityID = activityRef
+ s.ensureSummary(result)
+ s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)
+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing vulnerability results")
+ if saveErr := s.saveResult(scanCtx, result); saveErr != nil {
+ slog.WarnContext(scanCtx, "failed to save scan result", "error", saveErr)
+ }
+ slog.InfoContext(scanCtx, "background vulnerability scan completed",
+ "environmentId", envID, "imageId", imageID, "imageName", imageName, "durationMs", duration, "vulnerabilities", result.Summary.Total)
+ s.logScanEvent(scanCtx, event.EventTypeImageScan, "vulnerability_scan", envID, imageID, imageName, user, "")
+ s.completeVulnerabilityScanActivity(scanCtx, activityID, true, "")
+ }()
return pendingResult, nil
}
-// markStaleInternal fails a pending or scanning scan that outlived
-// scanStaleTimeout and returns the failure message and duration.
-func (s *Service) markStaleInternal(ctx context.Context, imageID string, status vulnerability.ScanStatus, scanTime time.Time) (string, int64, bool) {
+// markStale fails a pending or scanning scan that outlived scanStaleTimeout
+// and returns the failure message and duration.
+func (s *Service) markStale(ctx context.Context, imageID string, status vulnerability.ScanStatus, scanTime time.Time) (string, int64, bool) {
if status != vulnerability.ScanStatusScanning && status != vulnerability.ScanStatusPending {
return "", 0, false
}
@@ -262,12 +298,10 @@ func (s *Service) markStaleInternal(ctx context.Context, imageID string, status
errMsg := "Scan timed out or was interrupted. Please retry."
duration := time.Since(scanTime).Milliseconds()
- s.clearScanPhaseInternal(imageID)
-
+ s.scanPhases.Delete(imageID)
if s.records.MarkFailed == nil {
return errMsg, duration, true
}
-
if err := s.records.MarkFailed(ctx, imageID, errMsg, duration); err != nil {
slog.WarnContext(ctx, "failed to mark stale scan as failed", "error", err, "scanId", imageID)
}
@@ -279,11 +313,11 @@ func (s *Service) RefreshSummary(ctx context.Context, summary *vulnerability.Sca
if summary == nil {
return
}
- if errMsg, _, stale := s.markStaleInternal(ctx, summary.ImageID, summary.Status, summary.ScanTime); stale {
+ if errMsg, _, stale := s.markStale(ctx, summary.ImageID, summary.Status, summary.ScanTime); stale {
summary.Status = vulnerability.ScanStatusFailed
summary.Error = errMsg
}
- s.applyScanPhaseToSummaryInternal(summary)
+ summary.ScanPhase = s.resolveScanPhase(summary.ImageID, summary.Status)
}
// RefreshResult fails stale in-progress scans and attaches the live scan phase.
@@ -291,153 +325,18 @@ func (s *Service) RefreshResult(ctx context.Context, result *vulnerability.ScanR
if result == nil {
return
}
- if errMsg, duration, stale := s.markStaleInternal(ctx, result.ImageID, result.Status, result.ScanTime); stale {
+ if errMsg, duration, stale := s.markStale(ctx, result.ImageID, result.Status, result.ScanTime); stale {
result.Status = vulnerability.ScanStatusFailed
result.Error = errMsg
result.Duration = duration
}
- s.applyScanPhaseToResultInternal(result)
-}
-
-func (s *Service) scanImageInBackgroundInternal(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, activityID string, slotID int, releaseSlot func()) {
- var err error
- if releaseSlot == nil {
- slotID, releaseSlot, err = s.trivy.AcquireSlot(ctx, imageID, true)
- }
- if releaseSlot != nil {
- defer releaseSlot()
- if updateErr := s.records.MarkScanning(ctx, imageID); updateErr != nil {
- slog.WarnContext(ctx, "failed to mark queued scan as running", "imageId", imageID, "error", updateErr)
- }
- }
- defer s.clearScanPhaseInternal(imageID)
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseCreatingContainer)
- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseCreatingContainer, 10, "Preparing vulnerability scanner")
-
- slog.DebugContext(ctx,
- "starting background vulnerability scan",
- "environmentId", envID,
- "imageId", imageID,
- "imageName", imageName,
- )
-
- var scannerImage string
- if err == nil {
- scannerImage, err = s.trivy.EnsureImage(ctx)
- }
- if err != nil {
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)
- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Unable to prepare vulnerability scanner")
- errorMessage := "Trivy scanner is not available: " + err.Error()
- if errors.Is(err, common.ErrFeatureDisabled) {
- errorMessage = err.Error()
- }
- result := &vulnerability.ScanResult{
- ImageID: imageID,
- ImageName: imageName,
- ScanTime: time.Now(),
- Status: vulnerability.ScanStatusFailed,
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- Error: errorMessage,
- }
- if saveErr := s.saveResult(ctx, result); saveErr != nil {
- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)
- }
- slog.WarnContext(ctx,
- "background vulnerability scan failed before execution",
- "environmentId", envID,
- "imageId", imageID,
- "imageName", imageName,
- "error", result.Error,
- )
- s.completeVulnerabilityScanActivityInternal(ctx, activityID, false, result.Error)
- return
- }
-
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseScanningImage)
- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseScanningImage, 45, "Scanning image for vulnerabilities")
- startTime := time.Now()
- result, err := s.trivy.Scan(ctx, scannerImage, imageName, imageID, slotID)
- duration := time.Since(startTime).Milliseconds()
-
- if err != nil {
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)
- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing failed scan result")
- failedResult := &vulnerability.ScanResult{
- ImageID: imageID,
- ImageName: imageName,
- ScanTime: time.Now(),
- Status: vulnerability.ScanStatusFailed,
- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),
- Error: err.Error(),
- Duration: duration,
- }
- if saveErr := s.saveResult(ctx, failedResult); saveErr != nil {
- slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)
- }
- slog.WarnContext(ctx,
- "background vulnerability scan failed",
- "environmentId", envID,
- "imageId", imageID,
- "imageName", imageName,
- "durationMs", duration,
- "error", err,
- )
- s.logScanEvent(ctx, envID, imageID, imageName, user, false, err.Error())
- s.completeVulnerabilityScanActivityInternal(ctx, activityID, false, err.Error())
- return
- }
-
- result.Duration = duration
- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.ensureSummary(result)
- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)
- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing vulnerability results")
- if saveErr := s.saveResult(ctx, result); saveErr != nil {
- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)
- }
-
- slog.InfoContext(ctx,
- "background vulnerability scan completed",
- "environmentId", envID,
- "imageId", imageID,
- "imageName", imageName,
- "durationMs", duration,
- "vulnerabilities", result.Summary.Total,
- )
-
- s.logScanEvent(ctx, envID, imageID, imageName, user, true, "")
- s.completeVulnerabilityScanActivityInternal(ctx, activityID, true, "")
-}
-
-func (s *Service) startVulnerabilityScanActivityInternal(ctx context.Context, envID, imageID, imageName string, user *usertypes.Actor) string {
- if s.activityService == nil {
- return ""
- }
-
- scanActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{
- EnvironmentID: envID,
- Type: activitytypes.TypeVulnerabilityScan,
- ResourceType: new("image"),
- ResourceID: &imageID,
- ResourceName: &imageName,
- StartedBy: user,
- Progress: new(0),
- Step: string(vulnerability.ScanPhaseCreatingContainer),
- LatestMessage: "Vulnerability scan queued",
- })
- if err != nil {
- slog.WarnContext(ctx, "failed to create vulnerability scan activity", "error", err, "imageId", imageID)
- return ""
- }
- return scanActivity.ID
+ result.ScanPhase = s.resolveScanPhase(result.ImageID, result.Status)
}
-func (s *Service) updateVulnerabilityScanActivityInternal(ctx context.Context, activityID string, phase vulnerability.ScanPhase, progress int, message string) {
+func (s *Service) updateVulnerabilityScanActivity(ctx context.Context, activityID string, phase vulnerability.ScanPhase, progress int, message string) {
if s.activityService == nil || activityID == "" {
return
}
-
if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{
Level: activitytypes.MessageLevelInfo,
Message: message,
@@ -448,34 +347,36 @@ func (s *Service) updateVulnerabilityScanActivityInternal(ctx context.Context, a
}
}
-func (s *Service) completeVulnerabilityScanActivityInternal(ctx context.Context, activityID string, success bool, errMessage string) {
+func (s *Service) completeVulnerabilityScanActivity(ctx context.Context, activityID string, success bool, errMessage string) {
if s.activityService == nil || activityID == "" {
return
}
-
status := activitytypes.StatusSuccess
message := "Vulnerability scan completed"
var errorPtr *string
- if !success {
- if activitylib.CancelledByContext(ctx) {
- status = activitytypes.StatusCancelled
- message = "Vulnerability scan cancelled"
- } else {
- status = activitytypes.StatusFailed
- message = "Vulnerability scan failed"
- if strings.TrimSpace(errMessage) != "" {
- errorPtr = &errMessage
- message = errMessage
- }
- }
+ switch {
+ case success:
+ case activitylib.CancelledByContext(ctx):
+ status, message = activitytypes.StatusCancelled, "Vulnerability scan cancelled"
+ default:
+ status, message = activitytypes.StatusFailed, cmp.Or(errMessage, "Vulnerability scan failed")
+ errorPtr = mo.EmptyableToOption(errMessage).ToPointer()
}
-
if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errorPtr); err != nil {
slog.DebugContext(ctx, "failed to complete vulnerability scan activity", "activityId", activityID, "error", err)
}
}
-const scheduledScanMaxAge = 7 * 24 * time.Hour
+// logScanEvent records a manual or scheduled scan event; an empty errMsg means it succeeded.
+func (s *Service) logScanEvent(ctx context.Context, eventType event.EventType, action, envID, imageID, imageName string, user usertypes.Actor, errMsg string) {
+ metadata := database.JSON{"action": action, "imageId": imageID, "imageName": imageName, "success": errMsg == ""}
+ if errMsg != "" {
+ metadata["error"] = errMsg
+ }
+ if err := s.eventService.LogImageEvent(ctx, eventType, imageID, imageName, user.ID, user.Username, cmp.Or(envID, "0"), metadata); err != nil {
+ slog.WarnContext(ctx, "failed to log vulnerability scan event", "action", action, "error", err)
+ }
+}
// Policy describes the scan settings a stored result was produced with.
func (s *Service) Policy() string {
@@ -483,414 +384,254 @@ func (s *Service) Policy() string {
return fmt.Sprintf("ignoreUnfixed=%t;server=%s", s.trivy.IgnoreUnfixed(), serverURL)
}
-type scheduledScanTargetInternal struct {
- imageID string
- imageName string
-}
-
-type scheduledScanNotificationSummary struct {
- mu sync.Mutex
- totalFixable int
- imagesWithFixable int
- severityCounts map[string]int
- sampleCVEs []string
- seenCVEs map[string]struct{}
-}
-
-func newScheduledScanNotificationSummary() *scheduledScanNotificationSummary {
- return &scheduledScanNotificationSummary{
- severityCounts: map[string]int{
- "CRITICAL": 0,
- "HIGH": 0,
- "MEDIUM": 0,
- "LOW": 0,
- "UNKNOWN": 0,
- },
- sampleCVEs: make([]string, 0, 5),
- seenCVEs: make(map[string]struct{}),
- }
-}
-
-func (s *Service) runScheduledImageScanInternal(
- ctx context.Context,
- containerPool chan string,
- imageName, imageID, scannerVersion string,
- files map[string][]byte,
-) (*vulnerability.ScanResult, int64, error) {
- startTime := time.Now()
- slotID, releaseSlot, slotErr := s.trivy.AcquireSlot(ctx, "", true)
- if slotErr != nil {
- return nil, 0, slotErr
- }
- defer releaseSlot()
-
- var containerID string
- select {
- case containerID = <-containerPool:
- case <-ctx.Done():
- return nil, 0, ctx.Err()
- }
- defer func() { containerPool <- containerID }()
-
+// Targets lists the images due for a scheduled scan, skipping completed results
+// from the last week with the same scanner version and policy.
+func (s *Service) Targets(ctx context.Context) ([]Target, error) {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
- return nil, 0, err
- }
- result, scanErr := s.trivy.ExecScan(ctx, containerID, imageName, imageID, scannerVersion, slotID, files)
-
- return result, time.Since(startTime).Milliseconds(), scanErr
-}
-
-func (s *Service) saveScheduledFailedScanInternal(
- ctx context.Context,
- envID, imageID, imageName string,
- user usertypes.Actor,
- duration int64,
- scanErr error,
-) {
- failedResult := &vulnerability.ScanResult{
- ImageID: imageID,
- ImageName: imageName,
- ScanTime: time.Now(),
- Status: vulnerability.ScanStatusFailed,
- Error: scanErr.Error(),
- Duration: duration,
- }
- if saveErr := s.saveResult(ctx, failedResult); saveErr != nil {
- slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)
- }
- s.logScheduledScanEvent(ctx, envID, imageID, imageName, user, false, scanErr.Error())
-}
-
-func (s *Service) saveScheduledSuccessfulScanInternal(
- ctx context.Context,
- envID, imageID, imageName string,
- user usertypes.Actor,
- result *vulnerability.ScanResult,
- duration int64,
-) {
- result.Duration = duration
- s.ensureSummary(result)
- if saveErr := s.saveResult(ctx, result); saveErr != nil {
- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)
- }
- s.logScheduledScanEvent(ctx, envID, imageID, imageName, user, true, "")
-}
-
-func (s *Service) collectScheduledFixableSummaryInternal(
- ctx context.Context,
- envID string,
- result *vulnerability.ScanResult,
- summary *scheduledScanNotificationSummary,
-) {
- fixableVulns := s.fixedVulnerabilitiesForNotifications(ctx, envID, result)
- if len(fixableVulns) == 0 {
- return
- }
-
- summary.mu.Lock()
- defer summary.mu.Unlock()
- summary.imagesWithFixable++
- for i := range fixableVulns {
- v := fixableVulns[i]
- summary.totalFixable++
-
- sev := strings.ToUpper(strings.TrimSpace(string(v.Severity)))
- if _, ok := summary.severityCounts[sev]; !ok {
- sev = "UNKNOWN"
- }
- summary.severityCounts[sev]++
-
- cveID := strings.TrimSpace(v.VulnerabilityID)
- if cveID == "" {
- continue
- }
- if _, alreadyAdded := summary.seenCVEs[cveID]; alreadyAdded {
- continue
- }
- summary.seenCVEs[cveID] = struct{}{}
- if len(summary.sampleCVEs) < 5 {
- summary.sampleCVEs = append(summary.sampleCVEs, cveID)
- }
- }
-}
-
-func (s *Service) sendScheduledVulnerabilitySummaryInternal(
- ctx context.Context,
- scanned int,
- summary *scheduledScanNotificationSummary,
-) {
- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || summary.totalFixable == 0 || s.notificationService == nil {
- return
- }
-
- summaryDate := time.Now().UTC().Format("2006-01-02")
- payload := notification.VulnerabilityNotificationPayload{
- CVEID: "Daily Summary - " + summaryDate,
- Severity: fmt.Sprintf(
- "Critical:%d High:%d Medium:%d Low:%d Unknown:%d",
- summary.severityCounts["CRITICAL"],
- summary.severityCounts["HIGH"],
- summary.severityCounts["MEDIUM"],
- summary.severityCounts["LOW"],
- summary.severityCounts["UNKNOWN"],
- ),
- ImageName: fmt.Sprintf("%d image(s) scanned, %d with fixable vulnerabilities", scanned, summary.imagesWithFixable),
- FixedVersion: fmt.Sprintf("%d fixable vulnerability record(s)", summary.totalFixable),
- }
- if len(summary.sampleCVEs) > 0 {
- payload.PkgName = strings.Join(summary.sampleCVEs, ", ")
- }
-
- if err := s.notificationService.SendVulnerabilityNotification(ctx, payload); err != nil {
- slog.WarnContext(ctx, "failed to send daily vulnerability summary notification", "error", err)
- }
-}
-
-// ScanAllImages scans all Docker images for vulnerabilities. It is intended
-// for use by the scheduled vulnerability scan job. A single long-running Trivy
-// container is created and reused for every image via docker exec, which avoids
-// the overhead of creating/destroying a container per scan. The caller-supplied
-// user is recorded in the event log.
-func (s *Service) ScanAllImages(ctx context.Context, envID string, user usertypes.Actor, force bool) (scanned, failed int, err error) {
- if requireFeatureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); requireFeatureErr != nil {
- return 0, 0, requireFeatureErr
+ return nil, err
}
images, err := s.dockerService.ListImages(ctx)
if err != nil {
- return 0, 0, fmt.Errorf("failed to list images: %w", err)
- }
-
- targets := make([]scheduledScanTargetInternal, 0, len(images))
- for _, img := range images {
- imageID, imageName, shouldScan := scheduledScanImageTargetInternal(img)
- if shouldScan {
- targets = append(targets, scheduledScanTargetInternal{imageID: imageID, imageName: imageName})
- }
+ return nil, fmt.Errorf("failed to list images: %w", err)
}
-
scannerVersion := s.trivy.Version(ctx)
- if !force {
- targets, err = s.skipRecentlyScannedTargetsInternal(ctx, targets, scannerVersion)
- if err != nil {
- return 0, 0, err
+ targets := make([]Target, 0, len(images))
+ ids := make([]string, 0, len(images))
+ for _, img := range images {
+ imageName := slices.Concat(img.RepoTags, img.RepoDigests, []string{img.ID})[0]
+ // Skip intermediate and dangling images with no usable name.
+ if imageName != "<none>:<none>" && imageName != img.ID {
+ targets = append(targets, Target{ImageID: img.ID, ImageName: imageName, ScannerVersion: scannerVersion})
+ ids = append(ids, img.ID)
}
}
- if len(targets) == 0 {
- return 0, 0, nil
- }
-
- concurrency := s.trivy.ConcurrentScanContainers()
- containerIDs, files, cleanup, err := s.trivy.SetupBatchContainers(ctx, concurrency)
- if err != nil {
- return 0, 0, err
- }
- defer cleanup()
-
- containerPool := make(chan string, len(containerIDs))
- for _, containerID := range containerIDs {
- containerPool <- containerID
- }
-
- notificationSummary := newScheduledScanNotificationSummary()
- var scannedCount, failedCount atomic.Int64
- var featureStopped atomic.Bool
-
- g, gctx := errgroup.WithContext(ctx)
- g.SetLimit(concurrency)
- for _, target := range targets {
- g.Go(func() (workerErr error) {
- defer utils.RecoverToError(&workerErr, "scheduled vulnerability scan worker", "imageId", target.imageID)
- if gctx.Err() != nil {
- return gctx.Err()
- }
- if !s.settingsService.IsFeatureEnabled(gctx, features.VulnerabilityManagement) {
- featureStopped.Store(true)
- return nil
- }
- switch s.scanSingleImage(gctx, containerPool, target.imageID, target.imageName, scannerVersion, envID, user, notificationSummary, files) {
- case scanStatusSuccess:
- scannedCount.Add(1)
- case scanStatusFailed:
- failedCount.Add(1)
- case scanStatusSkipped:
- featureStopped.Store(true)
- }
- return nil
- })
- }
-
- err = g.Wait()
- scanned, failed = int(scannedCount.Load()), int(failedCount.Load())
- if err != nil {
- return scanned, failed, err
- }
-
- if featureStopped.Load() {
- return scanned, failed, fmt.Errorf("remaining scans stopped because feature %s was disabled: %w", features.VulnerabilityManagement, common.ErrFeatureDisabled)
- }
- s.sendScheduledVulnerabilitySummaryInternal(ctx, scanned, notificationSummary)
-
- return scanned, failed, nil
-}
-
-func (s *Service) skipRecentlyScannedTargetsInternal(ctx context.Context, targets []scheduledScanTargetInternal, scannerVersion string) ([]scheduledScanTargetInternal, error) {
if len(targets) == 0 {
return targets, nil
}
- ids := make([]string, 0, len(targets))
- for _, target := range targets {
- ids = append(ids, target.imageID)
- }
-
- var records []scanStateRowInternal
- if err := s.db.WithContext(ctx).
- Table(scansTableInternal).
+ var records []scanStateRow
+ if queryErr := s.db.WithContext(ctx).
+ Table(scansTable).
Select("id", "image_name", "status", "scan_time", "scanner_version", "scan_policy").
Where("id IN ?", ids).
- Find(&records).Error; err != nil {
- return nil, fmt.Errorf("failed to load existing scan records: %w", err)
+ Find(&records).Error; queryErr != nil {
+ return nil, fmt.Errorf("failed to load existing scan records: %w", queryErr)
}
-
cutoff := time.Now().Add(-scheduledScanMaxAge)
policy := s.Policy()
- recent := make(map[string]scanStateRowInternal, len(records))
+ recent := make(map[string]scanStateRow, len(records))
for _, record := range records {
if record.Status == string(vulnerability.ScanStatusCompleted) && record.ScannerVersion == scannerVersion && record.ScanPolicy == policy && record.ScanTime.After(cutoff) {
recent[record.ID] = record
}
}
-
- remaining := make([]scheduledScanTargetInternal, 0, len(targets))
- for _, target := range targets {
- record, ok := recent[target.imageID]
- if !ok {
- remaining = append(remaining, target)
- continue
- }
- if record.ImageName != target.imageName {
- if err := s.records.SetImageName(ctx, target.imageID, target.imageName); err != nil {
- slog.WarnContext(ctx, "failed to refresh scan record image name", "imageId", target.imageID, "error", err)
+ targets = slices.DeleteFunc(targets, func(target Target) bool {
+ record, ok := recent[target.ImageID]
+ if ok && record.ImageName != target.ImageName {
+ if renameErr := s.records.SetImageName(ctx, target.ImageID, target.ImageName); renameErr != nil {
+ slog.WarnContext(ctx, "failed to refresh scan record image name", "imageId", target.ImageID, "error", renameErr)
}
}
+ return ok
+ })
+ if len(recent) > 0 {
+ slog.InfoContext(ctx, "scheduled vulnerability scan: skipping recently scanned images", "skipped", len(recent), "maxAge", scheduledScanMaxAge.String())
}
+ return targets, nil
+}
- if skipped := len(targets) - len(remaining); skipped > 0 {
- slog.InfoContext(ctx, "scheduled vulnerability scan: skipping recently scanned images", "skipped", skipped, "maxAge", scheduledScanMaxAge.String())
+// joinPool returns the shared scan containers, creating them on first use; leave must be called when the scan ends.
+func (s *Service) joinPool(ctx context.Context) (*scanPool, func(), error) {
+ s.poolMu.Lock()
+ if s.pool == nil {
+ // Setup stays cancelable and bounded, since later scans wait on poolMu; removing the containers detaches on its own.
+ setupCtx, cancelSetup := context.WithTimeout(ctx, scanPoolSetupTimeout)
+ containerIDs, files, cleanup, err := s.trivy.SetupBatchContainers(setupCtx, s.trivy.ConcurrentScanContainers())
+ cancelSetup()
+ if err != nil {
+ s.poolMu.Unlock()
+ return nil, nil, common.Classify(common.ErrUnavailable, fmt.Errorf("prepare scan containers: %w", err))
+ }
+ s.pool = &scanPool{containers: make(chan string, len(containerIDs)), files: files, cleanup: cleanup, retired: make(chan struct{})}
+ for _, containerID := range containerIDs {
+ s.pool.containers <- containerID
+ }
}
-
- return remaining, nil
+ pool := s.pool
+ if pool.idle != nil {
+ pool.idle.Stop()
+ pool.idle = nil
+ }
+ pool.users++
+ s.poolMu.Unlock()
+ return pool, func() {
+ s.poolMu.Lock()
+ pool.users--
+ idle, retired := pool.users == 0 && s.pool == pool, pool.users == 0 && s.pool != pool
+ // An interrupted last scan, such as at shutdown, removes the pool now; the idle timer would not outlive the process.
+ if idle && ctx.Err() != nil {
+ s.pool, idle, retired = nil, false, true
+ }
+ if idle {
+ pool.idle = time.AfterFunc(scanPoolIdle, s.ReleaseScanPool)
+ }
+ s.poolMu.Unlock()
+ // The last scan out of a retired pool removes its containers, the unstoppable one included.
+ if retired {
+ pool.cleanup()
+ }
+ }, nil
}
-func (s *Service) fixedVulnerabilitiesForNotifications(
- ctx context.Context,
- envID string,
- result *vulnerability.ScanResult,
-) []vulnerability.Vulnerability {
- if result == nil || result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {
- return nil
+// ScanTarget scans one image in the shared scan containers and saves its result.
+// It returns common.ErrFeatureDisabled, without saving, once the feature is off.
+func (s *Service) ScanTarget(ctx context.Context, envID string, target Target) (Fixable, error) {
+ pool, leave, err := s.joinPool(ctx)
+ if err != nil {
+ return Fixable{}, err
}
+ defer leave()
- vulnerabilities := result.Vulnerabilities
- filtered, err := s.filterIgnored(ctx, envID, result.ImageID, vulnerabilities)
+ slog.InfoContext(ctx, "scheduled vulnerability scan: scanning image", "image", target.ImageName, "imageId", target.ImageID)
+ startTime := time.Now()
+ slotID, releaseSlot, err := s.trivy.AcquireSlot(ctx, "", true)
if err != nil {
- slog.WarnContext(ctx, "failed to filter ignored vulnerabilities for notifications", "error", err)
- } else {
- vulnerabilities = filtered
+ return Fixable{}, err
}
+ defer releaseSlot()
+ var containerID string
+ select {
+ case containerID = <-pool.containers:
+ case <-pool.retired:
+ return Fixable{}, common.Classify(common.ErrUnavailable, errors.New("scan containers were replaced; retrying"))
+ case <-ctx.Done():
+ return Fixable{}, ctx.Err()
+ }
+ defer func() {
+ // A cancel only detaches from the exec, so the container is restarted to stop Trivy before another scan reuses it.
+ if ctx.Err() != nil {
+ restartCtx, cancelRestart := context.WithTimeout(context.WithoutCancel(ctx), timeouts.GetDuration(0, timeouts.DefaultDockerAPI))
+ defer cancelRestart()
+ dockerClient, clientErr := s.dockerService.GetClient(restartCtx)
+ if clientErr == nil {
+ _, clientErr = dockerClient.ContainerRestart(restartCtx, containerID, client.ContainerRestartOptions{})
+ }
+ if clientErr != nil {
+ // Without this container the pool cannot serve scans, so later scans start a fresh one.
+ slog.WarnContext(restartCtx, "scheduled vulnerability scan: failed to stop a canceled scan; retiring its scan containers", "containerId", containerID, "error", clientErr)
+ s.poolMu.Lock()
+ select {
+ case <-pool.retired:
+ default:
+ close(pool.retired)
+ }
+ if s.pool == pool {
+ s.pool = nil
+ }
+ s.poolMu.Unlock()
+ return
+ }
+ }
+ pool.containers <- containerID
+ }()
+ if featureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); featureErr != nil {
+ return Fixable{}, featureErr
+ }
+ result, err := s.trivy.ExecScan(ctx, containerID, target.ImageName, target.ImageID, target.ScannerVersion, slotID, pool.files)
+ duration := time.Since(startTime).Milliseconds()
+ if err != nil {
+ failed := &vulnerability.ScanResult{
+ ImageID: target.ImageID, ImageName: target.ImageName, ScanTime: time.Now(),
+ Status: vulnerability.ScanStatusFailed, Error: err.Error(), Duration: duration,
+ }
+ if saveErr := s.saveResult(ctx, failed); saveErr != nil {
+ slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)
+ }
+ s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, err.Error())
+ return Fixable{}, err
+ }
+ result.Duration = duration
+ s.ensureSummary(result)
+ if saveErr := s.saveResult(ctx, result); saveErr != nil {
+ slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)
+ }
+ s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, "")
- fixable := make([]vulnerability.Vulnerability, 0, len(vulnerabilities))
- for i := range vulnerabilities {
- v := vulnerabilities[i]
+ fixable := Fixable{Severity: map[string]int{}}
+ if result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {
+ return fixable, nil
+ }
+ vulnerabilities, err := s.filterIgnored(ctx, envID, result.ImageID, result.Vulnerabilities)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to filter ignored vulnerabilities for notifications", "error", err)
+ vulnerabilities = result.Vulnerabilities
+ }
+ for _, v := range vulnerabilities {
if strings.TrimSpace(v.FixedVersion) == "" {
continue
}
- fixable = append(fixable, v)
+ fixable.Total++
+ fixable.Severity[strings.ToUpper(strings.TrimSpace(string(v.Severity)))]++
+ if cveID := strings.TrimSpace(v.VulnerabilityID); cveID != "" && len(fixable.CVEs) < 5 && !slices.Contains(fixable.CVEs, cveID) {
+ fixable.CVEs = append(fixable.CVEs, cveID)
+ }
}
-
- return fixable
+ return fixable, nil
}
-// logScanEvent logs a vulnerability scan event
-func (s *Service) logScanEvent(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, success bool, errMsg string) {
- metadata := database.JSON{
- "action": "vulnerability_scan",
- "imageId": imageID,
- "imageName": imageName,
- "success": success,
- }
- if errMsg != "" {
- metadata["error"] = errMsg
+// ReleaseScanPool removes the shared scan containers once no scan is using them.
+func (s *Service) ReleaseScanPool() {
+ s.poolMu.Lock()
+ pool := s.pool
+ if pool == nil || pool.users > 0 {
+ s.poolMu.Unlock()
+ return
}
-
- environmentID := cmp.Or(envID, "0")
-
- eventType := event.EventTypeImageScan
- if err := s.eventService.LogImageEvent(ctx, eventType, imageID, imageName, user.ID, user.Username, environmentID, metadata); err != nil {
- slog.WarnContext(ctx, "failed to log vulnerability scan event", "error", err)
+ s.pool = nil
+ if pool.idle != nil {
+ pool.idle.Stop()
}
+ s.poolMu.Unlock()
+ pool.cleanup()
}
-// logScheduledScanEvent logs a vulnerability scan event using the dedicated scheduled scan event type.
-func (s *Service) logScheduledScanEvent(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, success bool, errMsg string) {
- metadata := database.JSON{
- "action": "scheduled_vulnerability_scan",
- "imageId": imageID,
- "imageName": imageName,
- "success": success,
+// NotifySummary sends the scheduled scan's daily summary of fixable findings.
+func (s *Service) NotifySummary(ctx context.Context, scanned int, fixables []Fixable) {
+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.notificationService == nil {
+ return
}
- if errMsg != "" {
- metadata["error"] = errMsg
+ severity := map[string]int{"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "UNKNOWN": 0}
+ var total, images int
+ var sampleCVEs []string
+ for _, fixable := range fixables {
+ if fixable.Total == 0 {
+ continue
+ }
+ images++
+ total += fixable.Total
+ for sev, count := range fixable.Severity {
+ if _, known := severity[sev]; !known {
+ sev = "UNKNOWN"
+ }
+ severity[sev] += count
+ }
+ for _, cveID := range fixable.CVEs {
+ if len(sampleCVEs) < 5 && !slices.Contains(sampleCVEs, cveID) {
+ sampleCVEs = append(sampleCVEs, cveID)
+ }
+ }
}
-
- environmentID := cmp.Or(envID, "0")
-
- if err := s.eventService.LogImageEvent(ctx, event.EventTypeImageVulnerabilityScan, imageID, imageName, user.ID, user.Username, environmentID, metadata); err != nil {
- slog.WarnContext(ctx, "failed to log scheduled vulnerability scan event", "error", err)
+ if total == 0 {
+ return
}
-}
-
-type scanResultStatus int
-
-const (
- scanStatusSuccess scanResultStatus = iota
- scanStatusFailed
- scanStatusSkipped
-)
-
-func (s *Service) scanSingleImage(
- ctx context.Context,
- containerPool chan string,
- imageID, imageName, scannerVersion, envID string,
- user usertypes.Actor,
- notificationSummary *scheduledScanNotificationSummary,
- files map[string][]byte,
-) scanResultStatus {
- slog.InfoContext(ctx, "scheduled vulnerability scan: scanning image", "image", imageName, "imageId", imageID)
-
- result, duration, scanErr := s.runScheduledImageScanInternal(ctx, containerPool, imageName, imageID, scannerVersion, files)
- if scanErr != nil {
- if errors.Is(scanErr, common.ErrFeatureDisabled) {
- return scanStatusSkipped
- }
- s.saveScheduledFailedScanInternal(ctx, envID, imageID, imageName, user, duration, scanErr)
- return scanStatusFailed
+ payload := notification.VulnerabilityNotificationPayload{
+ CVEID: "Daily Summary - " + time.Now().UTC().Format("2006-01-02"),
+ Severity: fmt.Sprintf("Critical:%d High:%d Medium:%d Low:%d Unknown:%d", severity["CRITICAL"], severity["HIGH"], severity["MEDIUM"], severity["LOW"], severity["UNKNOWN"]),
+ ImageName: fmt.Sprintf("%d image(s) scanned, %d with fixable vulnerabilities", scanned, images),
+ FixedVersion: fmt.Sprintf("%d fixable vulnerability record(s)", total),
+ PkgName: strings.Join(sampleCVEs, ", "),
}
-
- s.saveScheduledSuccessfulScanInternal(ctx, envID, imageID, imageName, user, result, duration)
- s.collectScheduledFixableSummaryInternal(ctx, envID, result, notificationSummary)
- return scanStatusSuccess
-}
-
-func scheduledScanImageTargetInternal(img image.Summary) (imageID, imageName string, shouldScan bool) {
- imageID = img.ID
- imageName = imageID
- if len(img.RepoTags) > 0 {
- imageName = img.RepoTags[0]
- } else if len(img.RepoDigests) > 0 {
- imageName = img.RepoDigests[0]
+ if err := s.notificationService.SendVulnerabilityNotification(ctx, payload); err != nil {
+ slog.WarnContext(ctx, "failed to send daily vulnerability summary notification", "error", err)
}
-
- // Skip intermediate / dangling images with no usable name.
- return imageID, imageName, imageName != "<none>:<none>" && imageName != imageID
}
diff --git a/backend/internal/vulnerability/children/scan/service_test.go b/backend/internal/vulnerability/children/scan/service_test.go
index ad2bdc126..80ffde583 100644
--- a/backend/internal/vulnerability/children/scan/service_test.go
+++ b/backend/internal/vulnerability/children/scan/service_test.go
@@ -6,18 +6,18 @@ import (
"time"
"github.com/getarcaneapp/arcane/types/v2/features"
- "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/vulnerability"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {
+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {
t.Helper()
gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -32,9 +32,9 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database
}
func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {
- settingsSvc, db := featureSettingsInternal(t)
- require.NoError(t, db.Table("vulnerability_scans").AutoMigrate(&scanStateRowInternal{}))
- require.NoError(t, db.Table("vulnerability_scans").Create(&scanStateRowInternal{
+ settingsSvc, db := featureSettings(t)
+ require.NoError(t, db.Table("vulnerability_scans").AutoMigrate(&scanStateRow{}))
+ require.NoError(t, db.Table("vulnerability_scans").Create(&scanStateRow{
ID: "test-image", Status: string(vulnerability.ScanStatusCompleted), ScanTime: time.Now(),
}).Error)
svc := NewService(Dependencies{
@@ -45,21 +45,19 @@ func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {
},
})
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
- status := svc.scanSingleImage(t.Context(), nil, "test-image", "test:latest", "", "0", user.Actor{}, newScheduledScanNotificationSummary(), nil)
+ _, err := svc.ScanTarget(t.Context(), "0", Target{ImageID: "test-image", ImageName: "test:latest"})
- require.Equal(t, scanStatusSkipped, status)
- var record scanStateRowInternal
+ require.ErrorIs(t, err, common.ErrFeatureDisabled)
+ var record scanStateRow
require.NoError(t, db.Table("vulnerability_scans").First(&record, "id = ?", "test-image").Error)
require.Equal(t, string(vulnerability.ScanStatusCompleted), record.Status)
}
func TestDisabledFeatureSuppressesScheduledSummaryNotification(t *testing.T) {
- settingsSvc, _ := featureSettingsInternal(t)
+ settingsSvc, _ := featureSettings(t)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
svc := &Service{settingsService: settingsSvc, notificationService: &notification.NotificationService{}}
- summary := newScheduledScanNotificationSummary()
- summary.totalFixable = 1
require.NotPanics(t, func() {
- svc.sendScheduledVulnerabilitySummaryInternal(t.Context(), 1, summary)
+ svc.NotifySummary(t.Context(), 1, []Fixable{{Total: 1}})
})
}
diff --git a/backend/internal/vulnerability/model.go b/backend/internal/vulnerability/model.go
index 729a0ede4..d86a8e88d 100644
--- a/backend/internal/vulnerability/model.go
+++ b/backend/internal/vulnerability/model.go
@@ -110,7 +110,7 @@ func (VulnerabilityThreatIntel) TableName() string {
return "vulnerability_threat_intel"
}
-type vulnerabilityScanItemDetailsInternal struct {
+type vulnerabilityScanItemDetails struct {
Description string `json:"description,omitempty"`
References []string `json:"references,omitempty"`
CVSS *vulnerability.CVSSInfo `json:"cvss,omitempty"`
diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.go
index 10d83d071..fc66dfe25 100644
--- a/backend/internal/vulnerability/service.go
+++ b/backend/internal/vulnerability/service.go
@@ -9,13 +9,20 @@ import (
"net/http"
"time"
+ "github.com/getarcaneapp/arcane/types/v2"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/features"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/vulnerability"
+ "github.com/italypaleale/francis/builtin/workflow"
dockerregistry "github.com/moby/moby/api/types/registry"
+ kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -28,20 +35,36 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/store"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/dbutil"
)
+// ignoreIdentityQuery matches the ignore of one finding in one image and environment.
+const ignoreIdentityQuery = "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?"
+
+// scanParallelism caps concurrent scan tasks; Trivy slots apply the configured limit below it.
+const scanParallelism = 16
+
+// scanSlot is one scan task's output; failures travel as task errors.
+type scanSlot struct {
+ Skipped bool `json:"skipped,omitempty"`
+ Fixable scan.Fixable `json:"fixable"`
+}
+
// VulnerabilityService coordinates vulnerability scanning, findings, ignores,
// and risk, and owns the scan record and raw report persistence.
type VulnerabilityService struct {
db *database.DB
settingsService *settings.SettingsService
- scan *scan.Service
- query *query.Service
- store *store.Service
- ignores *ignores.Service
+ scan *scan.Service
+ query *query.Service
+ // scanWorkflow runs scheduled scans; see RegisterWorkflows.
+ scanWorkflow *flow.Workflow
+ store *store.Service
+ ignores *ignores.Service
}
// NewVulnerabilityService creates a new VulnerabilityService instance
@@ -65,46 +88,183 @@ func NewVulnerabilityService(
}
s.store = store.NewService(db, store.Records{
- Save: s.saveRecordInternal,
- SaveStatus: s.saveRecordStatusInternal,
- Delete: s.deleteRecordsInternal,
- DeleteReports: s.deleteReportsInternal,
- IDsExcept: s.recordIDsExceptInternal,
- DeleteExcept: s.deleteRecordsExceptInternal,
- EachMissingItems: s.eachRecordMissingItemsInternal,
- FixableCounts: s.fixableCountsInternal,
- SaveReport: s.saveReportInternal,
- SetFixableCount: s.setFixableCountInternal,
- DeleteItems: deleteItemsInternal,
- SaveItems: saveItemsInternal,
- CountKnownExploited: s.countKnownExploitedInternal,
- ReplaceKnownExploited: s.replaceKnownExploitedInternal,
- StaleEPSSCVEs: s.staleEPSSCVEsInternal,
- SaveEPSS: s.saveEPSSInternal,
- PruneThreatIntel: s.pruneThreatIntelInternal,
+ Save: s.saveRecord,
+ SaveStatus: func(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {
+ updates := map[string]any{
+ "image_name": result.ImageName,
+ "status": string(result.Status),
+ "scan_time": result.ScanTime,
+ "duration": result.Duration,
+ "critical_count": summary.Critical,
+ "high_count": summary.High,
+ "medium_count": summary.Medium,
+ "low_count": summary.Low,
+ "unknown_count": summary.Unknown,
+ "total_count": summary.Total,
+ "scanner_version": result.ScannerVersion,
+ "error": nil,
+ }
+ if errMessage != "" {
+ updates["error"] = errMessage
+ }
+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error
+ },
+ Delete: func(ctx context.Context, imageIDs []string) error {
+ if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {
+ return err
+ }
+ s.query.InvalidateRiskOverview()
+ return nil
+ },
+ DeleteReports: func(ctx context.Context, imageIDs []string) error {
+ return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error
+ },
+ IDsExcept: func(ctx context.Context, keepIDs []string) ([]string, error) {
+ scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})
+ if len(keepIDs) > 0 {
+ scope = scope.Where("id NOT IN ?", keepIDs)
+ }
+ var ids []string
+ err := scope.Pluck("id", &ids).Error
+ return ids, err
+ },
+ DeleteExcept: func(ctx context.Context, keepIDs []string) (int64, error) {
+ scope := s.db.WithContext(ctx).Where("1 = 1")
+ if len(keepIDs) > 0 {
+ scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)
+ }
+ result := scope.Delete(&VulnerabilityScanRecord{})
+ if result.Error != nil {
+ return 0, result.Error
+ }
+ return result.RowsAffected, nil
+ },
+ EachMissingItems: s.eachRecordMissingItems,
+ FixableCounts: func(ctx context.Context) (map[string]*int, error) {
+ var scans []VulnerabilityScanRecord
+ if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {
+ return nil, err
+ }
+ counts := make(map[string]*int, len(scans))
+ for i := range scans {
+ counts[scans[i].ID] = scans[i].FixableCount
+ }
+ return counts, nil
+ },
+ SaveReport: s.saveReport,
+ SetFixableCount: func(ctx context.Context, imageID string, count int) error {
+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error
+ },
+ DeleteItems: func(tx *gorm.DB, imageIDs []string) error {
+ return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error
+ },
+ SaveItems: saveItems,
+ CountKnownExploited: func(ctx context.Context) (int64, error) {
+ var count int64
+ err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error
+ return count, err
+ },
+ ReplaceKnownExploited: s.replaceKnownExploited,
+ StaleEPSSCVEs: func(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {
+ var cveIDs []string
+ err := s.db.WithContext(ctx).
+ Model(&VulnerabilityScanItem{}).
+ Distinct("vulnerability_scan_items.vulnerability_id").
+ Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").
+ Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").
+ Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).
+ Limit(limit).
+ Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error
+ return cveIDs, err
+ },
+ SaveEPSS: func(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {
+ batch := make([]VulnerabilityThreatIntel, 0, len(entries))
+ for i := range entries {
+ batch = append(batch, VulnerabilityThreatIntel(entries[i]))
+ }
+ return s.db.WithContext(ctx).Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "cve_id"}},
+ DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),
+ }).Create(&batch).Error
+ },
+ PruneThreatIntel: func(ctx context.Context) error {
+ return s.db.WithContext(ctx).
+ Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).
+ Delete(&VulnerabilityThreatIntel{}).Error
+ },
}, dockerService, settingsService, kvService, httpClient)
- s.query = query.NewService(db, cfg, dockerService, settingsService, s.loadResultInternal, s.refreshSummaryInternal, s.store.ThreatIntelStatus, s.filterIgnoredInternal)
+ s.query = query.NewService(db, cfg, dockerService, settingsService, s.loadResult, func(ctx context.Context, summary *vulnerability.ScanSummary) {
+ s.scan.RefreshSummary(ctx, summary)
+ }, s.store.ThreatIntelStatus, func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {
+ return s.ignores.FilterForImage(ctx, envID, imageID, vulns)
+ })
s.ignores = ignores.NewService(db, ignores.Records{
- ForImage: s.ignoresForImageInternal,
- Exists: s.ignoreExistsInternal,
- Get: s.getIgnoreInternal,
- Create: createIgnoresInternal,
- DeleteIdentity: s.deleteIgnoreIdentityInternal,
- DeleteIDs: deleteIgnoreIDsInternal,
- Page: s.ignorePageInternal,
- Changed: s.query.InvalidateRiskOverview,
+ ForImage: func(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {
+ var ignoreRecords []VulnerabilityIgnore
+ if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {
+ return nil, err
+ }
+ return ignoredVulnerabilities(ignoreRecords), nil
+ },
+ Exists: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {
+ var existing VulnerabilityIgnore
+ err := s.db.WithContext(ctx).Where(
+ ignoreIdentityQuery,
+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,
+ ).First(&existing).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, nil
+ }
+ return err == nil, err
+ },
+ Get: s.getIgnore,
+ Create: func(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {
+ records := make([]VulnerabilityIgnore, len(entries))
+ for i := range entries {
+ records[i] = VulnerabilityIgnore(entries[i])
+ }
+ if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {
+ return nil, err
+ }
+ return ignoredVulnerabilities(records), nil
+ },
+ DeleteIdentity: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {
+ return s.db.WithContext(ctx).Where(
+ ignoreIdentityQuery,
+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,
+ ).Delete(&VulnerabilityIgnore{}).Error
+ },
+ DeleteIDs: func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {
+ result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})
+ return result.RowsAffected, result.Error
+ },
+ Page: s.ignorePage,
+ Changed: s.query.InvalidateRiskOverview,
}, s.query.ActiveItemsQuery)
s.scan = scan.NewService(scan.Dependencies{
DB: db,
Records: scan.Records{
- MarkScanning: s.markScanningInternal,
- MarkFailed: s.markFailedInternal,
- SetImageName: s.setImageNameInternal,
+ MarkScanning: func(ctx context.Context, imageID string) error {
+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error
+ },
+ MarkFailed: func(ctx context.Context, imageID, message string, duration int64) error {
+ if s.db == nil {
+ return nil
+ }
+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{
+ "status": ScanStatusFailed,
+ "error": message,
+ "duration": duration,
+ }).Error
+ },
+ SetImageName: func(ctx context.Context, imageID, imageName string) error {
+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error
+ },
},
Config: cfg,
KV: kvService,
RegistryAuthConfigs: registryAuthConfigs,
- SaveReport: s.saveReportInternal,
+ SaveReport: s.saveReport,
Docker: dockerService,
Settings: settingsService,
Events: eventService,
@@ -129,11 +289,115 @@ func (s *VulnerabilityService) ScanImage(ctx context.Context, envID, imageID str
return s.scan.ScanImage(ctx, envID, imageID, user)
}
-// ScanAllImages runs the scheduled scan of every image.
-func (s *VulnerabilityService) ScanAllImages(ctx context.Context, envID string, user usertypes.Actor, force bool) (scanned, failed int, err error) {
- return s.scan.ScanAllImages(ctx, envID, user, force)
+// RegisterWorkflows defines the scheduled scan: discover due images, scan each
+// as its own retryable task, then notify. Call it before the host starts.
+func (s *VulnerabilityService) RegisterWorkflows(engine *flow.Engine) error {
+ scanWorkflow, err := engine.Define(flow.Definition{
+ Name: "vulnerability-scan",
+ Version: 1,
+ Fingerprint: "d221e3e86aded6a3afaed48abc267783425ff861c89accd834abf14148a754d3",
+ Concurrency: scanParallelism,
+ // Batches of large images can take hours; Trivy bounds each scan itself.
+ Timeout: 12 * time.Hour,
+ Activity: activitylib.StartRequest{
+ Type: activitytypes.TypeVulnerabilityScan,
+ ResourceType: new("images"),
+ ResourceName: new("Scheduled vulnerability scan"),
+ Step: "Discovering images",
+ LatestMessage: "Scheduled vulnerability scan started",
+ },
+ Labels: map[string]string{
+ "discover": "Discovering images",
+ "scan": "Scanning images",
+ "finalize": "Finishing vulnerability scan",
+ },
+ Steps: []workflow.StepSpec{
+ workflow.Step("discover", engine.Handler(func(ctx context.Context, _ flow.Task) (any, error) {
+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
+ return []scan.Target{}, nil
+ }
+ if deleted, err := s.CleanupOrphanedScanRecords(ctx); err != nil {
+ slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", err)
+ } else if deleted > 0 {
+ slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)
+ }
+ // A feature disabled since the check above fails this attempt; the retry returns no targets.
+ targets, err := s.scan.Targets(ctx)
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ return targets, nil
+ })),
+ // Slots and the shared container pool bound real parallelism to the configured limit.
+ workflow.ForEach("scan", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var target scan.Target
+ if err := t.DecodeItem(&target); err != nil {
+ return nil, err
+ }
+ fixable, err := s.scan.ScanTarget(ctx, types.LocalDockerEnvironmentID, target)
+ if errors.Is(err, common.ErrFeatureDisabled) {
+ return scanSlot{Skipped: true}, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return scanSlot{Fixable: fixable}, nil
+ }),
+ workflow.WithItemsFrom("discover"),
+ workflow.WithMaxParallel(scanParallelism),
+ workflow.WithFailurePolicy(workflow.TolerateFailures),
+ workflow.WithMaxAttempts(3),
+ workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),
+ workflow.Step("finalize", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ s.scan.ReleaseScanPool()
+ var targets []scan.Target
+ if err := t.DecodeOutput("discover", &targets); err != nil {
+ return nil, err
+ }
+ slots, err := flow.Results[scanSlot](t, "scan")
+ if err != nil {
+ return nil, err
+ }
+ outcome := scheduler.Outcome{Status: scheduler.Succeeded}
+ scanned, skipped := 0, 0
+ fixables := make([]scan.Fixable, 0, len(slots))
+ for index, slot := range slots {
+ switch {
+ case slot.Err != "":
+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ResourceType: "image", ID: targets[index].ImageName, Status: scheduler.Failed, Message: slot.Err})
+ case slot.Value.Skipped:
+ skipped++
+ default:
+ scanned++
+ fixables = append(fixables, slot.Value.Fixable)
+ }
+ }
+ outcome.Message = fmt.Sprintf("Vulnerability scan completed: %d scanned, %d failed", scanned, len(outcome.Targets))
+ switch {
+ case skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)):
+ outcome.Status = kit.Ternary(scanned > 0, scheduler.Partial, scheduler.Skipped)
+ outcome.Message = fmt.Sprintf("Remaining scans stopped because feature %s was disabled", features.VulnerabilityManagement)
+ case len(outcome.Targets) > 0:
+ outcome.Status = scheduler.Partial
+ }
+ if skipped == 0 {
+ s.scan.NotifySummary(ctx, scanned, fixables)
+ }
+ slog.InfoContext(ctx, "scheduled vulnerability scan completed", "scanned", scanned, "failed", len(outcome.Targets), "skipped", skipped)
+ return outcome, nil
+ }), workflow.WithInputFrom("discover")),
+ },
+ })
+ if err != nil {
+ return err
+ }
+ s.scanWorkflow = scanWorkflow
+ return nil
}
+// ScanWorkflow is the scheduled vulnerability scan workflow.
+func (s *VulnerabilityService) ScanWorkflow() *flow.Workflow { return s.scanWorkflow }
+
// GetScanSummariesByImageIDs retrieves scan summaries for multiple images.
func (s *VulnerabilityService) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []string) (map[string]*vulnerability.ScanSummary, error) {
return s.query.GetScanSummariesByImageIDs(ctx, imageIDs)
@@ -179,16 +443,8 @@ func (s *VulnerabilityService) ImportLegacyReportFiles(ctx context.Context) {
s.store.ImportLegacyReportFiles(ctx)
}
-func (s *VulnerabilityService) refreshSummaryInternal(ctx context.Context, summary *vulnerability.ScanSummary) {
- s.scan.RefreshSummary(ctx, summary)
-}
-
-func (s *VulnerabilityService) filterIgnoredInternal(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {
- return s.ignores.FilterForImage(ctx, envID, imageID, vulns)
-}
-
-// loadResultInternal reads the stored scan result for an image without insights.
-func (s *VulnerabilityService) loadResultInternal(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {
+// loadResult reads the stored scan result for an image without insights.
+func (s *VulnerabilityService) loadResult(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {
var record VulnerabilityScanRecord
err := s.db.WithContext(ctx).Where("id = ?", imageID).First(&record).Error
if err != nil {
@@ -198,17 +454,6 @@ func (s *VulnerabilityService) loadResultInternal(ctx context.Context, imageID s
return nil, fmt.Errorf("failed to get scan result: %w", err)
}
- result := convertRecordToResultInternal(ctx, &record)
- s.scan.RefreshResult(ctx, result)
- var reportCount int64
- if countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Error; countReportsErr == nil {
- result.HasReport = reportCount > 0
- }
- return result, nil
-}
-
-// convertRecordToResultInternal converts a database record to a ScanResult.
-func convertRecordToResultInternal(ctx context.Context, record *VulnerabilityScanRecord) *vulnerability.ScanResult {
result := &vulnerability.ScanResult{
ImageID: record.ID,
ImageName: record.ImageName,
@@ -232,19 +477,24 @@ func convertRecordToResultInternal(ctx context.Context, record *VulnerabilitySca
if len(record.Vulnerabilities) > 0 && record.Vulnerabilities[0] != "" {
var vulns []vulnerability.Vulnerability
- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err)
+ if decodeErr := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); decodeErr != nil {
+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", decodeErr)
} else {
result.Vulnerabilities = vulns
}
}
store.EnsureSummary(result)
- return result
+ s.scan.RefreshResult(ctx, result)
+ var reportCount int64
+ if countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Error; countReportsErr == nil {
+ result.HasReport = reportCount > 0
+ }
+ return result, nil
}
-// saveRecordInternal upserts the scan record of result inside tx.
-func (s *VulnerabilityService) saveRecordInternal(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {
+// saveRecord upserts the scan record of result inside tx.
+func (s *VulnerabilityService) saveRecord(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {
var vulnJSON database.StringSlice
if len(result.Vulnerabilities) > 0 {
vulnBytes, err := json.Marshal(result.Vulnerabilities)
@@ -288,66 +538,8 @@ func (s *VulnerabilityService) saveRecordInternal(tx *gorm.DB, result *vulnerabi
}).Create(&record).Error
}
-// saveRecordStatusInternal writes only the status, counts, and error of result.
-func (s *VulnerabilityService) saveRecordStatusInternal(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {
- updates := map[string]any{
- "image_name": result.ImageName,
- "status": string(result.Status),
- "scan_time": result.ScanTime,
- "duration": result.Duration,
- "critical_count": summary.Critical,
- "high_count": summary.High,
- "medium_count": summary.Medium,
- "low_count": summary.Low,
- "unknown_count": summary.Unknown,
- "total_count": summary.Total,
- "scanner_version": result.ScannerVersion,
- "error": nil,
- }
- if errMessage != "" {
- updates["error"] = errMessage
- }
- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error
-}
-
-func (s *VulnerabilityService) deleteRecordsInternal(ctx context.Context, imageIDs []string) error {
- if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {
- return err
- }
- s.query.InvalidateRiskOverview()
- return nil
-}
-
-func (s *VulnerabilityService) deleteReportsInternal(ctx context.Context, imageIDs []string) error {
- return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error
-}
-
-// recordIDsExceptInternal lists scan record IDs outside keepIDs; an empty keepIDs lists every record.
-func (s *VulnerabilityService) recordIDsExceptInternal(ctx context.Context, keepIDs []string) ([]string, error) {
- scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})
- if len(keepIDs) > 0 {
- scope = scope.Where("id NOT IN ?", keepIDs)
- }
- var ids []string
- err := scope.Pluck("id", &ids).Error
- return ids, err
-}
-
-// deleteRecordsExceptInternal deletes scan records outside keepIDs; an empty keepIDs deletes every record.
-func (s *VulnerabilityService) deleteRecordsExceptInternal(ctx context.Context, keepIDs []string) (int64, error) {
- scope := s.db.WithContext(ctx).Where("1 = 1")
- if len(keepIDs) > 0 {
- scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)
- }
- result := scope.Delete(&VulnerabilityScanRecord{})
- if result.Error != nil {
- return 0, result.Error
- }
- return result.RowsAffected, nil
-}
-
-// eachRecordMissingItemsInternal visits completed scans with findings but no finding rows.
-func (s *VulnerabilityService) eachRecordMissingItemsInternal(ctx context.Context, batchSize int, fn func(imageID, vulnerabilitiesJSON string)) error {
+// eachRecordMissingItems visits completed scans with findings but no finding rows.
+func (s *VulnerabilityService) eachRecordMissingItems(ctx context.Context, batchSize int, fn func(imageID, vulnerabilitiesJSON string)) error {
var batch []VulnerabilityScanRecord
return s.db.WithContext(ctx).
Select("id", "vulnerabilities").
@@ -365,20 +557,8 @@ func (s *VulnerabilityService) eachRecordMissingItemsInternal(ctx context.Contex
}).Error
}
-func (s *VulnerabilityService) fixableCountsInternal(ctx context.Context) (map[string]*int, error) {
- var scans []VulnerabilityScanRecord
- if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {
- return nil, err
- }
- counts := make(map[string]*int, len(scans))
- for i := range scans {
- counts[scans[i].ID] = scans[i].FixableCount
- }
- return counts, nil
-}
-
-// saveReportInternal stores the raw Trivy report of an image, replacing any earlier one.
-func (s *VulnerabilityService) saveReportInternal(ctx context.Context, imageID, data string) error {
+// saveReport stores the raw Trivy report of an image, replacing any earlier one.
+func (s *VulnerabilityService) saveReport(ctx context.Context, imageID, data string) error {
if s.db == nil {
return errors.New("database unavailable")
}
@@ -389,50 +569,12 @@ func (s *VulnerabilityService) saveReportInternal(ctx context.Context, imageID,
}).Create(&record).Error
}
-func (s *VulnerabilityService) setFixableCountInternal(ctx context.Context, imageID string, count int) error {
- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error
-}
-
-func (s *VulnerabilityService) markScanningInternal(ctx context.Context, imageID string) error {
- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error
-}
-
-func (s *VulnerabilityService) markFailedInternal(ctx context.Context, imageID, message string, duration int64) error {
- if s.db == nil {
- return nil
- }
- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{
- "status": ScanStatusFailed,
- "error": message,
- "duration": duration,
- }).Error
-}
-
-func (s *VulnerabilityService) setImageNameInternal(ctx context.Context, imageID, imageName string) error {
- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error
-}
-
const (
- scanItemInsertBatchSizeInternal = 500
- threatIntelUpsertBatchInternal = 500
+ scanItemInsertBatchSize = 500
+ threatIntelUpsertBatch = 500
)
-func deleteItemsInternal(tx *gorm.DB, imageIDs []string) error {
- return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error
-}
-
-func saveItemsInternal(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability) error {
- items, err := buildScanItemsInternal(imageID, vulns)
- if err != nil {
- return err
- }
- if storeItemsErr := tx.CreateInBatches(items, scanItemInsertBatchSizeInternal).Error; storeItemsErr != nil {
- return fmt.Errorf("failed to store scan items: %w", storeItemsErr)
- }
- return nil
-}
-
-func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability) ([]VulnerabilityScanItem, error) {
+func saveItems(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability) error {
items := make([]VulnerabilityScanItem, 0, len(vulns))
for i := range vulns {
v := &vulns[i]
@@ -452,7 +594,7 @@ func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability)
item.CVSSScore = new(score)
}
if v.Description != "" || len(v.References) > 0 || v.CVSS != nil || v.PublishedDate != nil || v.LastModifiedDate != nil {
- raw, err := json.Marshal(vulnerabilityScanItemDetailsInternal{
+ raw, err := json.Marshal(vulnerabilityScanItemDetails{
Description: v.Description,
References: v.References,
CVSS: v.CVSS,
@@ -460,23 +602,20 @@ func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability)
LastModifiedDate: v.LastModifiedDate,
})
if err != nil {
- return nil, fmt.Errorf("failed to marshal vulnerability details: %w", err)
+ return fmt.Errorf("failed to marshal vulnerability details: %w", err)
}
item.Details = new(string(raw))
}
items = append(items, item)
}
- return items, nil
-}
-
-func (s *VulnerabilityService) countKnownExploitedInternal(ctx context.Context) (int64, error) {
- var count int64
- err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error
- return count, err
+ if storeItemsErr := tx.CreateInBatches(items, scanItemInsertBatchSize).Error; storeItemsErr != nil {
+ return fmt.Errorf("failed to store scan items: %w", storeItemsErr)
+ }
+ return nil
}
-// replaceKnownExploitedInternal clears every stored KEV flag and upserts entries in one transaction.
-func (s *VulnerabilityService) replaceKnownExploitedInternal(ctx context.Context, entries []vulnerability.ThreatIntelEntry, now time.Time) error {
+// replaceKnownExploited clears every stored KEV flag and upserts entries in one transaction.
+func (s *VulnerabilityService) replaceKnownExploited(ctx context.Context, entries []vulnerability.ThreatIntelEntry, now time.Time) error {
rows := make([]VulnerabilityThreatIntel, 0, len(entries))
for i := range entries {
rows = append(rows, VulnerabilityThreatIntel(entries[i]))
@@ -490,63 +629,11 @@ func (s *VulnerabilityService) replaceKnownExploitedInternal(ctx context.Context
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "cve_id"}},
DoUpdates: clause.AssignmentColumns([]string{"known_exploited", "kev_date_added", "kev_due_date", "kev_ransomware", "updated_at"}),
- }).CreateInBatches(rows, threatIntelUpsertBatchInternal).Error
+ }).CreateInBatches(rows, threatIntelUpsertBatch).Error
})
}
-func (s *VulnerabilityService) staleEPSSCVEsInternal(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {
- var cveIDs []string
- err := s.db.WithContext(ctx).
- Model(&VulnerabilityScanItem{}).
- Distinct("vulnerability_scan_items.vulnerability_id").
- Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").
- Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").
- Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).
- Limit(limit).
- Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error
- return cveIDs, err
-}
-
-// saveEPSSInternal upserts EPSS scores without touching the KEV columns.
-func (s *VulnerabilityService) saveEPSSInternal(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {
- batch := make([]VulnerabilityThreatIntel, 0, len(entries))
- for i := range entries {
- batch = append(batch, VulnerabilityThreatIntel(entries[i]))
- }
- return s.db.WithContext(ctx).Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "cve_id"}},
- DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),
- }).Create(&batch).Error
-}
-
-// pruneThreatIntelInternal drops cached CVEs that are neither KEV-listed nor found by any scan.
-func (s *VulnerabilityService) pruneThreatIntelInternal(ctx context.Context) error {
- return s.db.WithContext(ctx).
- Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).
- Delete(&VulnerabilityThreatIntel{}).Error
-}
-
-func (s *VulnerabilityService) ignoresForImageInternal(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {
- var ignoreRecords []VulnerabilityIgnore
- if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {
- return nil, err
- }
- return ignoredVulnerabilitiesInternal(ignoreRecords), nil
-}
-
-func (s *VulnerabilityService) ignoreExistsInternal(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {
- var existing VulnerabilityIgnore
- err := s.db.WithContext(ctx).Where(
- "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?",
- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,
- ).First(&existing).Error
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return false, nil
- }
- return err == nil, err
-}
-
-func (s *VulnerabilityService) getIgnoreInternal(ctx context.Context, envID, ignoreID string) (*vulnerability.IgnoredVulnerability, bool, error) {
+func (s *VulnerabilityService) getIgnore(ctx context.Context, envID, ignoreID string) (*vulnerability.IgnoredVulnerability, bool, error) {
var record VulnerabilityIgnore
if err := s.db.WithContext(ctx).Where("id = ? AND environment_id = ?", ignoreID, envID).First(&record).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -554,45 +641,11 @@ func (s *VulnerabilityService) getIgnoreInternal(ctx context.Context, envID, ign
}
return nil, false, err
}
- ignore := ignoredVulnerabilityInternal(&record)
+ ignore := vulnerability.IgnoredVulnerability(record)
return &ignore, true, nil
}
-func createIgnoresInternal(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {
- records := make([]VulnerabilityIgnore, 0, len(entries))
- for i := range entries {
- ignore := &entries[i]
- records = append(records, VulnerabilityIgnore{
- ID: ignore.ID,
- EnvironmentID: ignore.EnvironmentID,
- ImageID: ignore.ImageID,
- VulnerabilityID: ignore.VulnerabilityID,
- PkgName: ignore.PkgName,
- InstalledVersion: ignore.InstalledVersion,
- Reason: ignore.Reason,
- CreatedBy: ignore.CreatedBy,
- CreatedAt: ignore.CreatedAt,
- })
- }
- if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {
- return nil, err
- }
- return ignoredVulnerabilitiesInternal(records), nil
-}
-
-func (s *VulnerabilityService) deleteIgnoreIdentityInternal(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {
- return s.db.WithContext(ctx).Where(
- "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?",
- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,
- ).Delete(&VulnerabilityIgnore{}).Error
-}
-
-func deleteIgnoreIDsInternal(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {
- result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})
- return result.RowsAffected, result.Error
-}
-
-func (s *VulnerabilityService) ignorePageInternal(
+func (s *VulnerabilityService) ignorePage(
ctx context.Context,
envID string,
order func(*gorm.DB) *gorm.DB,
@@ -609,27 +662,13 @@ func (s *VulnerabilityService) ignorePageInternal(
if err := scope.Offset(offset).Limit(limit).Find(&ignoreRecords).Error; err != nil {
return nil, 0, fmt.Errorf("failed to list ignored vulnerabilities: %w", err)
}
- return ignoredVulnerabilitiesInternal(ignoreRecords), total, nil
-}
-
-func ignoredVulnerabilityInternal(record *VulnerabilityIgnore) vulnerability.IgnoredVulnerability {
- return vulnerability.IgnoredVulnerability{
- ID: record.ID,
- EnvironmentID: record.EnvironmentID,
- ImageID: record.ImageID,
- VulnerabilityID: record.VulnerabilityID,
- PkgName: record.PkgName,
- InstalledVersion: record.InstalledVersion,
- Reason: record.Reason,
- CreatedBy: record.CreatedBy,
- CreatedAt: record.CreatedAt,
- }
+ return ignoredVulnerabilities(ignoreRecords), total, nil
}
-func ignoredVulnerabilitiesInternal(records []VulnerabilityIgnore) []vulnerability.IgnoredVulnerability {
+func ignoredVulnerabilities(records []VulnerabilityIgnore) []vulnerability.IgnoredVulnerability {
result := make([]vulnerability.IgnoredVulnerability, len(records))
for i := range records {
- result[i] = ignoredVulnerabilityInternal(&records[i])
+ result[i] = vulnerability.IgnoredVulnerability(records[i])
}
return result
}
diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.go
index 3326d5ba6..5817aaabc 100644
--- a/backend/internal/vulnerability/service_test.go
+++ b/backend/internal/vulnerability/service_test.go
@@ -13,6 +13,7 @@ import (
"time"
"github.com/getarcaneapp/arcane/types/v2/features"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/vulnerability"
"github.com/libtnb/sqlite"
@@ -21,13 +22,16 @@ import (
"go.getarcane.app/kit/pkg"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
)
func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {
@@ -42,7 +46,7 @@ func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {
func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
result := &vulnerability.ScanResult{
ImageID: "sha256:img-fixable",
@@ -65,7 +69,7 @@ func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {
func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testing.T) {
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
_, _, err := svc.query.ListVulnerabilities(t.Context(), "sha256:unknown", pagination.QueryParams{Limit: 20})
@@ -75,7 +79,7 @@ func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testi
func TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
records := []VulnerabilityScanRecord{
{ID: "sha256:img1", ImageName: "nginx:latest", Status: ScanStatusCompleted, ScanTime: time.Now()},
@@ -97,7 +101,7 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {
func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, nil))
require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, []string{}))
@@ -106,7 +110,7 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.
func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
vulns := []vulnerability.Vulnerability{
{
@@ -154,7 +158,7 @@ func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *tes
func TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
now := time.Now()
records := []VulnerabilityScanRecord{
@@ -178,7 +182,7 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(t
func TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAndNoFilter(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
now := time.Now()
records := []VulnerabilityScanRecord{
@@ -206,7 +210,7 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAn
func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
now := time.Now()
@@ -297,7 +301,7 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {
func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters(t *testing.T) {
ctx := t.Context()
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
+ svc := newTestService(db, nil)
vulnsA := []vulnerability.Vulnerability{
{VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},
@@ -356,9 +360,9 @@ func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFil
require.EqualValues(t, 3, page.GrandTotalItems)
}
-func riskTestServiceInternal(t *testing.T, httpClient *http.Client) (*VulnerabilityService, *settings.SettingsService, *database.DB) {
+func riskTestService(t *testing.T, httpClient *http.Client) (*VulnerabilityService, *settings.SettingsService, *database.DB) {
t.Helper()
- settingsSvc, db := featureSettingsInternal(t)
+ settingsSvc, db := featureSettings(t)
sqlDB, err := db.DB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
@@ -367,7 +371,7 @@ func riskTestServiceInternal(t *testing.T, httpClient *http.Client) (*Vulnerabil
return svc, settingsSvc, db
}
-func seedRiskScansInternal(t *testing.T, svc *VulnerabilityService) {
+func seedRiskScans(t *testing.T, svc *VulnerabilityService) {
t.Helper()
scans := []*vulnerability.ScanResult{
{
@@ -394,8 +398,8 @@ func seedRiskScansInternal(t *testing.T, svc *VulnerabilityService) {
}
func TestVulnerabilityService_GetRiskOverview(t *testing.T) {
- svc, _, db := riskTestServiceInternal(t, nil)
- seedRiskScansInternal(t, svc)
+ svc, _, db := riskTestService(t, nil)
+ seedRiskScans(t, svc)
require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2024-0002", KnownExploited: true, KEVRansomware: true, KEVDueDate: "2024-06-01"}).Error)
require.NoError(t, db.Create(&VulnerabilityIgnore{ID: "ignore-1", EnvironmentID: "0", ImageID: "sha256:web", VulnerabilityID: "CVE-2024-0003", PkgName: "zlib", InstalledVersion: "1.2.0"}).Error)
@@ -460,8 +464,8 @@ func TestVulnerabilityService_GetRiskOverview(t *testing.T) {
}
func TestVulnerabilityService_GetRiskOverview_ImageScoreMatchesInsights(t *testing.T) {
- svc, _, db := riskTestServiceInternal(t, nil)
- seedRiskScansInternal(t, svc)
+ svc, _, db := riskTestService(t, nil)
+ seedRiskScans(t, svc)
require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2024-0001", EPSSScore: new(0.25)}).Error)
overview, err := svc.query.GetRiskOverview(t.Context(), "0")
@@ -482,8 +486,8 @@ func TestVulnerabilityService_GetRiskOverview_ImageScoreMatchesInsights(t *testi
}
func TestVulnerabilityService_RecordRiskSnapshot(t *testing.T) {
- svc, _, db := riskTestServiceInternal(t, nil)
- seedRiskScansInternal(t, svc)
+ svc, _, db := riskTestService(t, nil)
+ seedRiskScans(t, svc)
now := time.Now().UTC()
require.NoError(t, db.Create(&[]query.VulnerabilityRiskSnapshot{
{SnapshotDate: now.AddDate(0, 0, -500).Format(time.DateOnly), ScoringVersion: vuln.ScoringVersion, RiskScore: 10, RiskBand: "low"},
@@ -513,7 +517,7 @@ func TestVulnerabilityService_RecordRiskSnapshot(t *testing.T) {
require.Equal(t, 3, overview.Drivers7dAgo.KnownExploited)
}
-// seedBulkIgnoreFixtureInternal stores nginx with 45 findings (more than one page) and redis with three findings covering every filter dimension.
+// seedBulkIgnoreFixture stores nginx with 45 findings (more than one page) and redis with three findings covering every filter dimension.
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
@@ -568,7 +572,7 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {
return srv.Client().Transport.RoundTrip(r)
})}
- svc, settingsSvc, db := riskTestServiceInternal(t, httpClient)
+ svc, settingsSvc, db := riskTestService(t, httpClient)
vulns := make([]vulnerability.Vulnerability, 0, 151)
for i := range 150 {
@@ -619,7 +623,7 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {
require.EqualValues(t, 1, kevRequests.Load(), "disabled intel makes no requests")
}
-func seedBulkIgnoreFixtureInternal(t *testing.T, db *database.DB, svc *VulnerabilityService) {
+func seedBulkIgnoreFixture(t *testing.T, db *database.DB, svc *VulnerabilityService) {
t.Helper()
nginx := make([]vulnerability.Vulnerability, 0, 45)
for i := range 45 {
@@ -665,7 +669,7 @@ func seedBulkIgnoreFixtureInternal(t *testing.T, db *database.DB, svc *Vulnerabi
svc.BackfillScanItems(t.Context())
}
-func activeCountInternal(t *testing.T, svc *VulnerabilityService, envID string) int64 {
+func activeCount(t *testing.T, svc *VulnerabilityService, envID string) int64 {
t.Helper()
_, page, err := svc.query.ListAllVulnerabilities(t.Context(), envID, pagination.QueryParams{Start: 0, Limit: 20})
require.NoError(t, err)
@@ -689,8 +693,8 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
db := setupVulnerabilityScanTestDB(t)
- svc := newTestServiceInternal(db, nil)
- seedBulkIgnoreFixtureInternal(t, db, svc)
+ svc := newTestService(db, nil)
+ seedBulkIgnoreFixture(t, db, svc)
before, err := svc.ActionableCountExcludingIgnored(ctx)
require.NoError(t, err)
@@ -699,8 +703,8 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {
affected, err := svc.ignores.BulkIgnoreVulnerabilities(ctx, "env-1", &tc.filters, "bob")
require.NoError(t, err)
require.Equal(t, tc.expected, affected)
- require.EqualValues(t, 48-tc.expected, activeCountInternal(t, svc, "env-1"))
- require.EqualValues(t, 48, activeCountInternal(t, svc, "env-2"), "other environments keep their own view")
+ require.EqualValues(t, 48-tc.expected, activeCount(t, svc, "env-1"))
+ require.EqualValues(t, 48, activeCount(t, svc, "env-2"), "other environments keep their own view")
var actionableIgnored int64
require.NoError(t, db.Model(&VulnerabilityIgnore{}).
@@ -723,7 +727,7 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {
restored, err := svc.ignores.BulkUnignoreVulnerabilities(ctx, "env-1", &tc.filters)
require.NoError(t, err)
require.Equal(t, tc.expected, restored)
- require.EqualValues(t, 48, activeCountInternal(t, svc, "env-1"))
+ require.EqualValues(t, 48, activeCount(t, svc, "env-1"))
after, err = svc.ActionableCountExcludingIgnored(ctx)
require.NoError(t, err)
require.Equal(t, before, after)
@@ -731,7 +735,7 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {
}
}
-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {
+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {
t.Helper()
gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -746,15 +750,15 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database
}
func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T) {
- settingsSvc, db := featureSettingsInternal(t)
- svc := newTestServiceInternal(db, settingsSvc)
+ settingsSvc, db := featureSettings(t)
+ svc := newTestService(db, settingsSvc)
require.NoError(t, db.Create(&VulnerabilityScanRecord{ID: "test-image", ImageName: "test:latest", Status: ScanStatusCompleted, ScanTime: time.Now()}).Error)
require.NoError(t, db.Create(&VulnerabilityReportRecord{ImageID: "test-image", Data: "stored report"}).Error)
require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
_, err := svc.ScanImage(t.Context(), "0", "test-image", user.Actor{})
require.ErrorIs(t, err, common.ErrFeatureDisabled)
- _, _, err = svc.ScanAllImages(t.Context(), "0", user.Actor{}, false)
+ _, err = svc.scan.Targets(t.Context())
require.ErrorIs(t, err, common.ErrFeatureDisabled)
summaries, err := svc.GetScanSummariesByImageIDs(t.Context(), []string{"test-image"})
require.NoError(t, err)
@@ -772,7 +776,23 @@ func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T)
require.Equal(t, "stored report", report.Data)
}
-// newTestServiceInternal wires a VulnerabilityService and its children over db.
-func newTestServiceInternal(db *database.DB, settingsService *settings.SettingsService) *VulnerabilityService {
+// newTestService wires a VulnerabilityService and its children over db.
+func newTestService(db *database.DB, settingsService *settings.SettingsService) *VulnerabilityService {
return NewVulnerabilityService(db, nil, nil, nil, settingsService, nil, nil, nil, nil, nil)
}
+
+func TestScanWorkflowSkipsWhenFeatureDisabled(t *testing.T) {
+ settingsSvc, db := featureSettings(t)
+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ svc := newTestService(db, settingsSvc)
+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))
+ harness.Start(t)
+ flowtest.AssertDefinitions(t, harness)
+ require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))
+
+ outcome, err := harness.Engine.Run(t.Context(), svc.ScanWorkflow(), nil, activitylib.StartRequest{}, nil)
+ require.NoError(t, err)
+ require.Equal(t, scheduler.Skipped, outcome.Status)
+ require.Len(t, outcome.Steps, 3)
+}
diff --git a/backend/pkg/libarcane/activity/requests.go b/backend/pkg/libarcane/activity/requests.go
index 6fc11eeff..a50f0c481 100644
--- a/backend/pkg/libarcane/activity/requests.go
+++ b/backend/pkg/libarcane/activity/requests.go
@@ -40,6 +40,8 @@ type SlotWaiter interface {
}
type StartRequest struct {
+ // ID is an optional caller-chosen ID; starting an existing ID returns that row.
+ ID string
EnvironmentID string
// BatchID groups activities spawned by one logical user action. When nil,
// the batch ID attached to the request context (if any) is used instead.
diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.go
index b7107d7ea..46916f152 100644
--- a/backend/pkg/scheduler/analytics_job.go
+++ b/backend/pkg/scheduler/analytics_job.go
@@ -26,9 +26,13 @@ const (
AnalyticsJobName = "analytics-heartbeat"
defaultHeartbeatEndpoint = "https://checkin.getarcane.app/heartbeat"
devHeartbeatEndpoint = "http://localhost:8080/heartbeat"
+ analyticsHeartbeatNextAttemptKey = "analytics.heartbeat.next_attempt_at"
+ // analyticsHeartbeatLastAttemptKey is the window older releases kept; it is carried forward once.
analyticsHeartbeatLastAttemptKey = "analytics.heartbeat.last_attempt_at"
analyticsHeartbeatDedupeWindow = 24 * time.Hour
- analyticsHeartbeatCheckSchedule = "0 0 * * * *"
+ // analyticsHeartbeatRetryDelay spaces retries of a failed send, which the hourly check then picks up.
+ analyticsHeartbeatRetryDelay = time.Hour
+ analyticsHeartbeatCheckSchedule = "0 0 * * * *"
)
type AnalyticsJob struct {
@@ -79,7 +83,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- allowed, err := j.claimHeartbeatAttemptWindowInternal(ctx)
+ allowed, err := j.claimHeartbeat(ctx)
if err != nil {
slog.ErrorContext(ctx, "failed to acquire analytics heartbeat send window", "error", err)
return schedulertypes.Outcome{}, err
@@ -106,22 +110,8 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
return schedulertypes.Outcome{}, err
}
- slog.InfoContext(
- ctx,
- "sending analytics heartbeat",
- "jobName",
- AnalyticsJobName,
- "version",
- payload.Version,
- "instanceId",
- payload.InstanceID,
- "serverType",
- payload.ServerType,
- "heartbeatUrl",
- j.heartbeatURL,
- "env",
- j.cfg.Environment,
- )
+ slog.InfoContext(ctx, "sending analytics heartbeat", "version", payload.Version, "instanceId", payload.InstanceID,
+ "serverType", payload.ServerType, "heartbeatUrl", j.heartbeatURL, "env", j.cfg.Environment)
_, err = backoff.Retry(
ctx,
@@ -158,7 +148,12 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
if bodyText != "" {
details = append(details, "response="+bodyText)
}
- return struct{}{}, fmt.Errorf("analytics heartbeat request failed: %s", strings.Join(details, "; "))
+ failure := fmt.Errorf("analytics heartbeat request failed: %s", strings.Join(details, "; "))
+ // Retrying a rejection within seconds only spends more of the endpoint's limit.
+ if resp.StatusCode < http.StatusInternalServerError {
+ return struct{}{}, backoff.Permanent(failure)
+ }
+ return struct{}{}, failure
}
return struct{}{}, nil
},
@@ -166,26 +161,15 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
backoff.WithMaxTries(3),
)
if err != nil {
- slog.ErrorContext(ctx, "analytics heartbeat failed", "error", err)
+ slog.ErrorContext(ctx, "analytics heartbeat failed; retrying after the retry delay", "error", err, "retryDelay", analyticsHeartbeatRetryDelay)
+ retryAt := j.now().UTC().Add(analyticsHeartbeatRetryDelay).Format(time.RFC3339Nano)
+ if retryErr := j.kvService.Set(ctx, analyticsHeartbeatNextAttemptKey, retryAt); retryErr != nil {
+ slog.WarnContext(ctx, "failed to shorten analytics heartbeat window after a failed send", "error", retryErr)
+ }
return schedulertypes.Outcome{}, err
}
- slog.InfoContext(
- ctx,
- "analytics heartbeat sent successfully",
- "jobName",
- AnalyticsJobName,
- "version",
- payload.Version,
- "instanceId",
- payload.InstanceID,
- "serverType",
- payload.ServerType,
- "heartbeatUrl",
- j.heartbeatURL,
- "env",
- j.cfg.Environment,
- )
+ slog.InfoContext(ctx, "analytics heartbeat sent successfully", "version", payload.Version, "instanceId", payload.InstanceID, "serverType", payload.ServerType)
return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
@@ -194,58 +178,41 @@ func (j *AnalyticsJob) Reschedule(ctx context.Context) error {
return nil
}
-func (j *AnalyticsJob) claimHeartbeatAttemptWindowInternal(ctx context.Context) (bool, error) {
+// claimHeartbeat reports whether a heartbeat is due and, if so, schedules the next one a day out. Claiming before
+// sending keeps restarts from sending twice; a failed send moves the next attempt up to the retry delay.
+func (j *AnalyticsJob) claimHeartbeat(ctx context.Context) (bool, error) {
if j.kvService == nil {
return false, errors.New("analytics heartbeat kv service is not configured")
}
-
j.runMu.Lock()
defer j.runMu.Unlock()
now := j.now().UTC()
- rawLastAttemptAt, ok, err := j.kvService.Get(ctx, analyticsHeartbeatLastAttemptKey)
+ rawNextAttemptAt, ok, err := j.kvService.Get(ctx, analyticsHeartbeatNextAttemptKey)
if err != nil {
return false, fmt.Errorf("failed to load analytics heartbeat attempt state: %w", err)
}
-
- if ok {
- lastAttemptAt, parseErr := time.Parse(time.RFC3339Nano, rawLastAttemptAt)
- if parseErr != nil {
- slog.WarnContext(
- ctx,
- "invalid analytics heartbeat attempt timestamp; resetting dedupe window",
- "key",
- analyticsHeartbeatLastAttemptKey,
- "value",
- rawLastAttemptAt,
- "error",
- parseErr,
- )
- } else {
- nextEligibleAt := lastAttemptAt.Add(analyticsHeartbeatDedupeWindow)
- if now.Before(nextEligibleAt) {
- slog.InfoContext(
- ctx,
- "skipping analytics heartbeat; already attempted within dedupe window",
- "jobName",
- AnalyticsJobName,
- "lastAttemptAt",
- lastAttemptAt,
- "nextEligibleAt",
- nextEligibleAt,
- )
- return false, nil
- }
+ nextAttemptAt, parseErr := time.Parse(time.RFC3339Nano, rawNextAttemptAt)
+ if !ok {
+ // An upgrade from a release that stored the last attempt keeps that window instead of sending again.
+ rawLastAttemptAt, hadLast, lastErr := j.kvService.Get(ctx, analyticsHeartbeatLastAttemptKey)
+ if lastErr != nil {
+ return false, fmt.Errorf("failed to load analytics heartbeat attempt state: %w", lastErr)
+ }
+ if lastAttemptAt, lastParseErr := time.Parse(time.RFC3339Nano, rawLastAttemptAt); hadLast && lastParseErr == nil {
+ ok, parseErr, nextAttemptAt = true, nil, lastAttemptAt.Add(analyticsHeartbeatDedupeWindow)
}
}
-
- // Persist the attempt window before sending on purpose: the product behavior is
- // best-effort at-most-once-per-24h check-ins, which avoids duplicate heartbeats
- // after restarts or partially completed outbound requests.
- if setErr := j.kvService.Set(ctx, analyticsHeartbeatLastAttemptKey, now.Format(time.RFC3339Nano)); setErr != nil {
+ if ok && parseErr != nil {
+ slog.WarnContext(ctx, "invalid analytics heartbeat attempt timestamp; resetting window", "value", rawNextAttemptAt, "error", parseErr)
+ }
+ if ok && parseErr == nil && now.Before(nextAttemptAt) {
+ slog.InfoContext(ctx, "skipping analytics heartbeat; next one is not due yet", "nextAttemptAt", nextAttemptAt)
+ return false, nil
+ }
+ if setErr := j.kvService.Set(ctx, analyticsHeartbeatNextAttemptKey, now.Add(analyticsHeartbeatDedupeWindow).Format(time.RFC3339Nano)); setErr != nil {
return false, fmt.Errorf("failed to persist analytics heartbeat attempt state: %w", setErr)
}
-
return true, nil
}
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 1ee282ad5..8af0ac596 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -1,7 +1,6 @@
package scheduler
import (
- "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -104,15 +103,7 @@ func (j *AutoHealJob) ShouldSchedule(ctx context.Context) bool {
}
func (j *AutoHealJob) Schedule(ctx context.Context) string {
- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "autoHealInterval", autoHealDefaultSchedule), autoHealDefaultSchedule)
-
- parser := scheduleutil.Parser()
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalidSchedule", schedule, "error", err)
- return autoHealDefaultSchedule
- }
-
- return schedule
+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "autoHealInterval", autoHealDefaultSchedule), autoHealDefaultSchedule, "auto-heal")
}
func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
diff --git a/backend/pkg/scheduler/auto_patch_job.go b/backend/pkg/scheduler/auto_patch_job.go
index fa3500e5e..2f8b54f75 100644
--- a/backend/pkg/scheduler/auto_patch_job.go
+++ b/backend/pkg/scheduler/auto_patch_job.go
@@ -1,90 +1,33 @@
package scheduler
import (
- "cmp"
"context"
- "errors"
- "log/slog"
- "github.com/getarcaneapp/arcane/types/v2"
"github.com/getarcaneapp/arcane/types/v2/features"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/getarcaneapp/arcane/types/v2/user"
- "go.getarcane.app/kit/pkg"
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
)
-const AutoPatchJobName = "auto-patch"
-
-var autoPatchSystemUser = user.Actor{
- Username: "System",
-}
-
-// AutoPatchJob periodically patches images whose latest vulnerability scan
-// found fixable OS package vulnerabilities. It is opt-in via the
-// "imageAutoPatchEnabled" setting.
-type AutoPatchJob struct {
- imageService *image.ImageService
- settingsService *settings.SettingsService
-}
-
-// NewAutoPatchJob creates a new AutoPatchJob.
-func NewAutoPatchJob(imageService *image.ImageService, settingsService *settings.SettingsService) *AutoPatchJob {
- return &AutoPatchJob{
- imageService: imageService,
- settingsService: settingsService,
- }
-}
-
-func (j *AutoPatchJob) Name() string {
- return AutoPatchJobName
-}
-
-func (j *AutoPatchJob) ShouldSchedule(ctx context.Context) bool {
- return j.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && j.settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false)
-}
-
-// Schedule returns the cron expression for the job. Defaults to daily at 3 AM.
-func (j *AutoPatchJob) Schedule(ctx context.Context) string {
- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "imageAutoPatchInterval", "0 0 3 * * *"), "0 0 3 * * *")
-
- parser := scheduleutil.Parser()
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for auto-patch, using default", "invalidSchedule", schedule, "error", err)
- return "0 0 3 * * *"
- }
-
- return schedule
-}
-
-func (j *AutoPatchJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
- if !j.ShouldSchedule(ctx) {
- slog.DebugContext(ctx, "scheduled image patching disabled; skipping run")
- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
- }
-
- slog.InfoContext(ctx, "scheduled image patching started")
+const (
+ AutoPatchJobName = "auto-patch"
+ autoPatchDefaultCron = "0 0 3 * * *"
+)
- patched, skipped, err := j.imageService.PatchFlaggedImages(ctx, types.LocalDockerEnvironmentID, autoPatchSystemUser)
- if errors.Is(err, common.ErrFeatureDisabled) {
- status := kit.Ternary(patched > 0, schedulertypes.Partial, schedulertypes.Skipped)
- return schedulertypes.Outcome{Status: status, Message: err.Error()}, nil
+// NewAutoPatchJob runs the durable auto-patch workflow. It is opt-in via the
+// "imageAutoPatchEnabled" setting and defaults to daily at 03:00.
+func NewAutoPatchJob(engine *flow.Engine, imageService *image.ImageService, settingsService *settings.SettingsService) *flow.Job {
+ return &flow.Job{
+ Engine: engine,
+ Workflow: imageService.PatchWorkflow(),
+ JobName: AutoPatchJobName,
+ ScheduleFn: func(ctx context.Context) string {
+ return scheduleutil.Or(ctx, settingsService.GetStringSetting(ctx, "imageAutoPatchInterval", autoPatchDefaultCron), autoPatchDefaultCron, AutoPatchJobName)
+ },
+ ShouldRunFn: func(ctx context.Context) bool {
+ return settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false)
+ },
}
- if err != nil {
- slog.ErrorContext(ctx, "scheduled image patching failed", "error", err)
- if patched > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.Partial}, err
- }
- return schedulertypes.Outcome{}, err
- }
-
- slog.InfoContext(ctx, "scheduled image patching completed",
- "patched", patched,
- "skipped", skipped,
- )
- return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index e77612ba2..732a104fc 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -1,254 +1,32 @@
package scheduler
import (
- "cmp"
"context"
- "errors"
- "log/slog"
- "strings"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- updatertypes "github.com/getarcaneapp/arcane/types/v2/updater"
kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
)
-const autoUpdateAdmissionScopeInternal = "auto-update"
-
-// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,
-// kept as an interface so the overlap guard is testable with a fake.
-type pendingUpdateApplierInternal interface {
- ApplyPending(ctx context.Context, options updatertypes.Options) (*updatertypes.Result, error)
-}
-
-type AutoUpdateJob struct {
- updaterService pendingUpdateApplierInternal
- settingsService *settings.SettingsService
- admissionGate *runs.Admission
-}
-
-func NewAutoUpdateJob(updaterService *updater.UpdaterService, settingsService *settings.SettingsService, admissionGate *runs.Admission) (*AutoUpdateJob, error) {
- if admissionGate == nil {
- return nil, errors.New("auto-update admission gate unavailable")
- }
- return &AutoUpdateJob{
- updaterService: updaterService,
- settingsService: settingsService,
- admissionGate: admissionGate,
- }, nil
-}
-
-func (j *AutoUpdateJob) Name() string {
- return "auto-update"
-}
-
-func (j *AutoUpdateJob) ShouldSchedule(ctx context.Context) bool {
- enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)
- pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)
- return enabled && pollingEnabled
-}
-
-func (j *AutoUpdateJob) Schedule(ctx context.Context) string {
- s := j.settingsService.GetStringSetting(ctx, "autoUpdateInterval", "0 0 0 * * *")
- return kit.Ternary(s == "", "0 0 0 * * *", s)
-}
-
-func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
- options := updatertypes.Options{}
- unresolvedTargets := false
- if previous, ok := jobcontext.Run(ctx); ok && (previous.AttemptCount > 1 || len(previous.Outcome.Targets) > 0) {
- var priorOutcome schedulertypes.Outcome
- options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)
- if priorOutcome.Status != "" {
- priorOutcome.ActivityID = cmp.Or(priorOutcome.ActivityID, previous.Outcome.ActivityID)
- if priorOutcome.Status == schedulertypes.Failed && previous.Outcome.Message != "" {
- priorOutcome.Message += ": " + previous.Outcome.Message
- }
- return priorOutcome, nil
- }
- }
-
- enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)
- pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)
- if !enabled || !pollingEnabled {
- if len(options.ResourceIds) > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Auto-update is disabled; the remaining targets were not updated"}, nil
- }
- slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run",
- "autoUpdate", enabled, "pollingEnabled", pollingEnabled)
- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
- }
-
- lease, admitted, err := j.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})
- if err != nil {
- slog.ErrorContext(ctx, "auto-update admission failed", "error", err)
- return schedulertypes.Outcome{}, err
- }
- if !admitted {
- if len(options.ResourceIds) > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Another update is active; the remaining targets were not updated"}, nil
- }
- slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
- }
- defer lease.Release(ctx)
-
- slog.InfoContext(ctx, "auto-update run started")
-
- result, err := j.updaterService.ApplyPending(ctx, options)
- if err != nil {
- slog.ErrorContext(ctx, "auto-update run failed", "err", err)
- if result == nil {
- return schedulertypes.Outcome{}, err
- }
- }
- if result == nil {
- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Updater returned no operation result"}, nil
- }
-
- return autoUpdateOutcomeInternal(ctx, result, unresolvedTargets, err)
-}
-
-func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {
- slog.InfoContext(ctx, "rescheduling auto-update job in new scheduler; currently requires restart")
- return nil
-}
-
-func (j *AutoUpdateJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
- if service, ok := j.updaterService.(*updater.UpdaterService); ok {
- return service.ReconcilePending(ctx, previous)
- }
- confirmed := confirmedAutoUpdateInternal(previous)
- if confirmed.Status != schedulertypes.Succeeded {
- confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion"
- if previous.Outcome.Message != "" {
- confirmed.Message += ": " + previous.Outcome.Message
- }
- }
- return confirmed, nil
-}
-
-func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Options, schedulertypes.Outcome, bool) {
- options := updatertypes.Options{}
- unresolvedTargets := true
- for _, target := range previous.Outcome.Targets {
- if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Partial) {
- unresolvedTargets = false
- }
- }
- confirmed := confirmedAutoUpdateInternal(previous)
- if confirmed.Status == schedulertypes.Succeeded {
- return options, confirmed, false
- }
- hasResourceTargets := false
- for _, target := range previous.Outcome.Targets {
- if target.ID == "auto-update" {
- continue
- }
- hasResourceTargets = true
- if target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {
- continue
- }
- if target.ResourceType != "container" || target.Status != schedulertypes.Failed || strings.TrimSpace(target.ID) == "" {
- unresolvedTargets = true
- continue
- }
- options.ResourceIds = append(options.ResourceIds, target.ID)
- }
- if hasResourceTargets && len(options.ResourceIds) == 0 {
- if unresolvedTargets {
- return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Remaining targets have no confirmed completion or safe retry", Targets: previous.Outcome.Targets}, true
- }
- return options,
- schedulertypes.Outcome{
- Status: schedulertypes.Failed,
- Message: "Target results do not confirm full-batch completion",
- ActivityID: previous.Outcome.ActivityID,
- Targets: previous.Outcome.Targets,
- },
- true
- }
- if !hasResourceTargets {
- return options,
- schedulertypes.Outcome{
- Status: schedulertypes.Failed,
- Message: "The previous update has no confirmed retry targets",
- ActivityID: previous.Outcome.ActivityID,
- Targets: previous.Outcome.Targets,
- },
- true
- }
- if len(options.ResourceIds) > 0 {
- options.Type = "container"
- }
- return options, schedulertypes.Outcome{}, unresolvedTargets
-}
-
-// ValidateRetry restricts replay to explicitly failed container updates.
-func (j *AutoUpdateJob) ValidateRetry(_ context.Context, previous schedulertypes.Run) error {
- options, outcome, _ := autoUpdateRetryOptionsInternal(previous)
- if outcome.Status != "" || len(options.ResourceIds) == 0 {
- return errors.New("auto-update has no safely retryable failed containers")
- }
- return nil
-}
-
-func autoUpdateOutcomeInternal(ctx context.Context, result *updatertypes.Result, unresolvedTargets bool, err error) (schedulertypes.Outcome, error) {
- slog.InfoContext(ctx, "auto-update run completed",
- "checked", result.Checked,
- "updated", result.Updated,
- "restarted", result.Restarted,
- "skipped", result.Skipped,
- "failed", result.Failed,
- )
- outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
- if result.ActivityID != nil {
- outcome.ActivityID = *result.ActivityID
- }
- for _, item := range result.Items {
- status := kit.Ternary(item.Status == "failed", schedulertypes.Failed, schedulertypes.Succeeded)
- if item.Status == "skipped" {
- status = schedulertypes.Skipped
- }
- outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error, ActivityID: outcome.ActivityID})
- }
- if result.Failed > 0 || err != nil || unresolvedTargets {
- outcome.Status = schedulertypes.Partial
- outcome.Message = "Some updates failed"
- if err != nil {
- outcome.Message += ": " + err.Error()
- }
- }
- if unresolvedTargets {
- outcome.Status = schedulertypes.Failed
- outcome.Message = "Some target outcomes could not be confirmed"
- }
- if outcomeErr, ok := errors.AsType[*schedulertypes.OutcomeError](err); ok {
- outcome.Status = outcomeErr.Outcome.Status
- outcome.Message = outcomeErr.Outcome.Message
- outcome.Targets = nil
- }
- return outcome, err
-}
-
-func confirmedAutoUpdateInternal(previous schedulertypes.Run) schedulertypes.Outcome {
- confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")
- for _, target := range previous.Outcome.Targets {
- if target.ID == "auto-update" && target.ResourceType != "update-batch" {
- confirmed.Status = schedulertypes.Failed
- }
- }
- if confirmed.Status == schedulertypes.Succeeded {
- for _, target := range previous.Outcome.Targets {
- if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
- confirmed.Status = schedulertypes.Failed
- break
- }
- }
+const autoUpdateDefaultSchedule = "0 0 0 * * *"
+
+// NewAutoUpdateJob runs the durable auto-update workflow on the configured schedule.
+// Interrupted runs that cannot resume fall back to frozen-plan recovery.
+func NewAutoUpdateJob(engine *flow.Engine, updaterService *updater.UpdaterService, settingsService *settings.SettingsService) *flow.Job {
+ return &flow.Job{
+ Engine: engine,
+ Workflow: updaterService.AutoUpdateWorkflow(),
+ JobName: "auto-update",
+ ScheduleFn: func(ctx context.Context) string {
+ schedule := settingsService.GetStringSetting(ctx, "autoUpdateInterval", autoUpdateDefaultSchedule)
+ return kit.Ternary(schedule == "", autoUpdateDefaultSchedule, schedule)
+ },
+ ShouldRunFn: func(ctx context.Context) bool {
+ return settingsService.GetBoolSetting(ctx, "autoUpdate", false) && settingsService.GetBoolSetting(ctx, "pollingEnabled", true)
+ },
+ FallbackFn: updaterService.ReconcilePending,
+ ValidateRetryFn: updaterService.ValidateAutoUpdateRetry,
}
- return confirmed
}
diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.go
index 7fae0cc42..b0392d247 100644
--- a/backend/pkg/scheduler/auto_update_job_test.go
+++ b/backend/pkg/scheduler/auto_update_job_test.go
@@ -1,23 +1,17 @@
package scheduler
import (
- "context"
- "sync/atomic"
"testing"
- "time"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/getarcaneapp/arcane/types/v2/updater"
"github.com/stretchr/testify/require"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
)
func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T) {
ctx := t.Context()
_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- job, err := NewAutoUpdateJob(nil, settingsSvc, newTestAdmissionGateInternal(t))
- require.NoError(t, err)
+ job := NewAutoUpdateJob(nil, &updater.UpdaterService{}, settingsSvc)
require.False(t, job.ShouldSchedule(ctx))
@@ -30,181 +24,3 @@ func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))
require.False(t, job.ShouldSchedule(ctx))
}
-
-type blockingApplierFakeInternal struct {
- calls atomic.Int32
- started chan struct{}
- release chan struct{}
-}
-
-func (f *blockingApplierFakeInternal) ApplyPending(context.Context, updater.Options) (*updater.Result, error) {
- f.calls.Add(1)
- select {
- case f.started <- struct{}{}:
- default:
- }
- <-f.release
- return &updater.Result{}, nil
-}
-
-func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
- ctx := t.Context()
- _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", true))
-
- applier := &blockingApplierFakeInternal{
- started: make(chan struct{}, 1),
- release: make(chan struct{}),
- }
- job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc, admissionGate: newTestAdmissionGateInternal(t)}
-
- firstDone := make(chan struct{})
- go func() {
- _, runErr1 := job.Run(ctx)
- if runErr1 != nil {
- t.Errorf("first auto update job run failed: %v", runErr1)
- close(firstDone)
- return
- }
- close(firstDone)
- }()
- <-applier.started
-
- // Overlapping tick returns immediately without a second ApplyPending.
- _, runErr2 := job.Run(ctx)
- require.NoError(t, runErr2)
- require.Equal(t, int32(1), applier.calls.Load())
-
- close(applier.release)
- select {
- case <-firstDone:
- case <-time.After(time.Second):
- require.FailNow(t, "first auto-update run did not finish")
- }
-
- // The guard resets once the run finishes.
- _, runErr3 := job.Run(ctx)
- require.NoError(t, runErr3)
- require.Equal(t, int32(2), applier.calls.Load())
- for _, tc := range []struct {
- name string
- targets []schedulertypes.TargetOutcome
- retryIDs []string
- status schedulertypes.RunStatus
- unresolved bool
- }{
- {name: "missing progress", unresolved: true, status: schedulertypes.Failed},
- {
- name: "image pull alone",
- unresolved: true,
- targets: []schedulertypes.TargetOutcome{{
- ID: "image",
- ResourceType: "image",
- Status: schedulertypes.Succeeded,
- }},
- status: schedulertypes.Failed,
- },
- {
- name: "failed containers only",
- unresolved: true,
- targets: []schedulertypes.TargetOutcome{
- {
- ID: "done",
- ResourceType: "container",
- Status: schedulertypes.Succeeded,
- },
- {
- ID: "failed",
- ResourceType: "container",
- Status: schedulertypes.Failed,
- },
- {
- ID: "image",
- ResourceType: "image",
- Status: schedulertypes.Failed,
- },
- },
- retryIDs: []string{"failed"},
- },
- {
- name: "completed failed batch retry",
- targets: []schedulertypes.TargetOutcome{
- {
- ID: "auto-update",
- ResourceType: "update-batch",
- Status: schedulertypes.Partial,
- },
- {
- ID: "failed",
- ResourceType: "container",
- Status: schedulertypes.Failed,
- },
- },
- retryIDs: []string{"failed"},
- },
- {
- name: "scoped retry cannot confirm original batch",
- targets: []schedulertypes.TargetOutcome{
- {
- ID: "auto-update",
- ResourceType: "update-retry",
- Status: schedulertypes.Succeeded,
- },
- {
- ID: "done",
- ResourceType: "container",
- Status: schedulertypes.Succeeded,
- },
- },
- status: schedulertypes.Failed,
- unresolved: true,
- },
- {
- name: "completed batch",
- targets: []schedulertypes.TargetOutcome{{
- ID: "auto-update",
- ResourceType: "update-batch",
- Status: schedulertypes.Succeeded,
- }},
- status: schedulertypes.Succeeded,
- },
- } {
- t.Run(tc.name, func(t *testing.T) {
- previous := schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: tc.targets}}
- options, outcome, unresolved := autoUpdateRetryOptionsInternal(previous)
- require.Equal(t, tc.unresolved, unresolved)
- require.Equal(t, tc.retryIDs, options.ResourceIds)
- require.Equal(t, tc.status, outcome.Status)
- if len(tc.retryIDs) == 0 {
- require.Error(t, job.ValidateRetry(ctx, previous))
- } else {
- require.NoError(t, job.ValidateRetry(ctx, previous))
- }
- reconciled, err := job.Reconcile(ctx, previous)
- require.NoError(t, err)
- if tc.status == schedulertypes.Succeeded {
- require.Equal(t, schedulertypes.Succeeded, reconciled.Status)
- } else {
- require.Equal(t, schedulertypes.Failed, reconciled.Status)
- }
- })
- }
-
- require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))
- retryCtx := jobcontext.WithExecution(
- ctx,
- schedulertypes.Run{
- AttemptCount: 2,
- Outcome: schedulertypes.Outcome{Targets: []schedulertypes.TargetOutcome{{
- ID: "failed",
- ResourceType: "container",
- Status: schedulertypes.Failed,
- }}},
- },
- nil,
- )
- outcome, err := job.Run(retryCtx)
- require.NoError(t, err)
- require.Equal(t, schedulertypes.Failed, outcome.Status)
- require.Equal(t, int32(2), applier.calls.Load())
-}
diff --git a/backend/pkg/scheduler/docker_client_refresh_job.go b/backend/pkg/scheduler/docker_client_refresh_job.go
index b8aac5532..70649eefa 100644
--- a/backend/pkg/scheduler/docker_client_refresh_job.go
+++ b/backend/pkg/scheduler/docker_client_refresh_job.go
@@ -1,7 +1,6 @@
package scheduler
import (
- "cmp"
"context"
"log/slog"
@@ -37,15 +36,7 @@ func (j *DockerClientRefreshJob) Name() string {
}
func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {
- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "dockerClientRefreshInterval", dockerClientRefreshDefaultSchedule), dockerClientRefreshDefaultSchedule)
-
- parser := scheduleutil.Parser()
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for Docker client refresh, using default", "invalidSchedule", schedule, "error", err)
- return dockerClientRefreshDefaultSchedule
- }
-
- return schedule
+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "dockerClientRefreshInterval", dockerClientRefreshDefaultSchedule), dockerClientRefreshDefaultSchedule, "docker-client-refresh")
}
func (j *DockerClientRefreshJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
diff --git a/backend/pkg/scheduler/entityjobs/entityjobs.go b/backend/pkg/scheduler/entityjobs/entityjobs.go
index 68366150c..928b2f02a 100644
--- a/backend/pkg/scheduler/entityjobs/entityjobs.go
+++ b/backend/pkg/scheduler/entityjobs/entityjobs.go
@@ -1,7 +1,4 @@
-// Package entityjobs holds the per-entity dynamic-job registry shared by the
-// services that schedule one job per database row (GitOps syncs, environment
-// health checks). It lives beside the scheduler rather than inside it because
-// package scheduler imports the services that would use this registry.
+// Package entityjobs registers one dynamic scheduler job per database row, such as a GitOps sync or a backup policy.
package entityjobs
import (
@@ -17,12 +14,8 @@ import (
// GitOpsSyncJobPrefix is shared by the GitOps registry and environment cleanup.
const GitOpsSyncJobPrefix = "gitops-sync:"
-// Registry owns one dynamic scheduler job per entity ID, plus the admission
-// gate that keeps a single run of any one entity in flight at a time.
-//
-// The scheduler and the app lifecycle context arrive post-construction via
-// SetScheduler: the manager wires them during bootstrap, and agent mode leaves
-// them nil so every registration becomes a no-op.
+// Registry owns one job per entity ID and admits one run per entity at a time.
+// Without SetScheduler, as in agent mode, every registration is a no-op.
type Registry struct {
jobPrefix string
admissionScope string
@@ -38,15 +31,10 @@ func New(jobPrefix, admissionScope string) *Registry {
return &Registry{jobPrefix: jobPrefix, admissionScope: admissionScope}
}
-// SetScheduler injects the job scheduler, the admission gate and the app
-// lifecycle context. The lifecycle context is what scheduled runs execute on,
-// so they outlive the request or bootstrap goroutine that registered them.
+// SetScheduler injects the scheduler, the admission gate, and the app lifecycle context scheduled runs execute on.
//
//nolint:contextcheck // scheduled runs must capture the app lifecycle context, not request contexts
-func (r *Registry) SetScheduler(ctx context.Context,
- scheduler schedulertypes.DynamicScheduler,
- admissionGate *runs.Admission,
-) error {
+func (r *Registry) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {
if scheduler == nil || admissionGate == nil {
return fmt.Errorf("%s scheduler dependencies unavailable", r.admissionScope)
}
@@ -62,12 +50,10 @@ func (r *Registry) SetScheduler(ctx context.Context,
// Enabled reports whether a scheduler has been injected.
func (r *Registry) Enabled() bool { return r.scheduler != nil }
-// Scheduler exposes the injected scheduler for the rare caller that must touch
-// a job this registry does not own (nil when none was injected).
+// Scheduler returns the injected scheduler, or nil.
func (r *Registry) Scheduler() schedulertypes.DynamicScheduler { return r.scheduler }
-// Context returns the context scheduled work should run on: the app lifecycle
-// context when one was injected, otherwise a cancel-detached copy of ctx.
+// Context returns the app lifecycle context, or a cancel-detached copy of ctx before one is injected.
func (r *Registry) Context(ctx context.Context) context.Context {
if r.lifecycleCtx != nil {
return r.lifecycleCtx
@@ -82,29 +68,24 @@ func (r *Registry) Context(ctx context.Context) context.Context {
func (r *Registry) JobName(entityID string) string { return r.jobPrefix + entityID }
// Register adds (or replaces) the dynamic job for entityID.
-func (r *Registry) Register(ctx context.Context,
- entityID string,
- schedule func(context.Context) string,
- run func(context.Context) (schedulertypes.Outcome,
- error),
- reconcile ...func(context.Context,
- schedulertypes.Run) (schedulertypes.Outcome,
- error),
+func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string,
+ run func(context.Context) (schedulertypes.Outcome, error), reconcile ...func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error),
) {
+ job := &schedulertypes.GenericJob{JobName: r.JobName(entityID), ScheduleFn: schedule, RunFn: run}
+ if len(reconcile) > 0 {
+ job.ReconcileFn = reconcile[0]
+ }
+ r.Add(ctx, job)
+}
+
+// Add adds (or replaces) a prebuilt job, such as a flow.Job named with JobName.
+func (r *Registry) Add(ctx context.Context, job schedulertypes.Job) {
if r.scheduler == nil {
return
}
schedulerCtx := r.Context(ctx)
- job := &schedulertypes.GenericJob{
- JobName: r.JobName(entityID),
- ScheduleFn: schedule,
- RunFn: run,
- }
- if len(reconcile) > 0 {
- job.ReconcileFn = reconcile[0]
- }
if err := r.scheduler.AddJob(schedulerCtx, job); err != nil {
- slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.JobName, "error", err)
+ slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.Name(), "error", err)
}
}
@@ -116,8 +97,7 @@ func (r *Registry) Unregister(ctx context.Context, entityID string) {
r.scheduler.RemoveJob(r.Context(ctx), r.JobName(entityID))
}
-// TryAcquire admits at most one in-flight run per entity ID. It refuses
-// immediately when a run is already active.
+// TryAcquire admits one in-flight run per entity ID and refuses at once while one is active.
func (r *Registry) TryAcquire(ctx context.Context, entityID string) (*runs.Lease, bool, error) {
return r.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: r.admissionScope, ID: entityID})
}
diff --git a/backend/pkg/scheduler/flow/flow.go b/backend/pkg/scheduler/flow/flow.go
new file mode 100644
index 000000000..f9c46e845
--- /dev/null
+++ b/backend/pkg/scheduler/flow/flow.go
@@ -0,0 +1,630 @@
+// Package flow runs durable multi-step jobs as Francis workflows inside
+// coordinator runs. The run stays the source of truth for status, the Jobs UI,
+// and Activity; the workflow instance is its execution graph.
+//
+// Migrating a job:
+// 1. Write each step as a func(ctx context.Context, t flow.Task) (any, error), inline in the definition
+// unless it is shared or too complex to inline.
+// Return common.Classify(common.ErrUnavailable or common.ErrTimeout, err) to retry; other errors fail the task.
+// Tasks run at least once, so confirm external effects before repeating them.
+// 2. Keep inputs and outputs small: keys, not lists of data, since Francis ships the instance input with
+// every task. Keep data in domain tables and never put secrets in a payload.
+// 3. Define the workflow with Engine.Define in a RegisterWorkflows method called from the service's DI
+// provider. The last step returns scheduler.Outcome.
+// 4. Register a flow.Job in place of the old job.
+// 5. Cover the definition with flowtest.AssertDefinitions. On any graph or policy change, bump Version and
+// set Fingerprint to the value the test reports.
+package flow
+
+import (
+ "cmp"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+ "uuid"
+
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/builtin/workflow"
+
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+)
+
+const (
+ defaultMaxPayloadSize = 256 << 10
+ defaultMaxJournalSize = 4 << 20
+ // stepWriteInterval bounds how often step progress rewrites the run record.
+ stepWriteInterval = 500 * time.Millisecond
+ // evidenceActorType stores the target progress of standalone instances, which have no run to hold it.
+ evidenceActorType = "workflow-evidence"
+)
+
+var (
+ // ErrNotResumable reports that a run has no workflow instance this host can serve.
+ ErrNotResumable = errors.New("workflow instance cannot be resumed")
+ // errInstanceFinished releases a root's cancellation once the instance ends.
+ errInstanceFinished = errors.New("workflow instance finished")
+)
+
+type (
+ // Definition declares a workflow. Bump Version and update Fingerprint on any
+ // graph or policy change; flowtest.AssertDefinitions reports the new value.
+ Definition struct {
+ Name string
+ Version int
+ Fingerprint string
+ Concurrency int
+ Timeout time.Duration
+ // Activity is the template for the activity each root instance owns; an empty Type means none.
+ Activity activitylib.StartRequest
+ // Labels name steps on the activity; steps without one show their name.
+ Labels map[string]string
+ // Steps use Francis step builders with handlers wrapped by Engine.Handler.
+ // The last step returns scheduler.Outcome.
+ Steps []workflow.StepSpec
+ Options []workflow.Option
+ }
+
+ // Workflow is a defined workflow registered on the engine's host.
+ Workflow struct {
+ def Definition
+ wf *workflow.Workflow
+ svc *workflow.WorkflowService
+ }
+
+ // Activities is the activity service surface the engine drives.
+ Activities interface {
+ activitylib.Service
+ activitylib.MessageAppender
+ activitylib.Tracker
+ UpdateActivity(ctx context.Context, activityID string, req activitylib.UpdateRequest) (*activitytypes.Activity, error)
+ AwaitActivitySlotBounded(ctx context.Context, activityID, environmentID string) error
+ }
+
+ // Engine defines workflows, runs them inside coordinator runs, and owns their activities.
+ Engine struct {
+ runtime *francis.Runtime
+ coordinator *runs.Coordinator
+ activities Activities
+ ctx context.Context
+ cancel context.CancelFunc
+ mu sync.Mutex
+ workflows []*Workflow
+ monitors map[string]*instanceMonitor
+ roots map[string]rootCancellation
+ evidence utils.KeyedMutex
+ // slots serializes reacquiring a root activity's slot, so its sibling tasks never wait on the slot it holds.
+ slots utils.KeyedMutex
+ // tracked holds each starting activity's cancel registration until its monitor takes it over.
+ tracked map[string]context.Context
+ // admitted holds the runs whose executor authorized them and waits on them; other runs' tasks wait for reconcile.
+ admitted map[string]struct{}
+ // ready closes once startup recovery has protected standalone work; until then standalone tasks wait.
+ ready chan struct{}
+ // submitted holds the standalone instances this process started; tasks of others are recovered.
+ submitted map[string]struct{}
+ readyOnce sync.Once
+ }
+
+ // rootCancellation interrupts every running task of one root instance.
+ rootCancellation struct {
+ ctx context.Context
+ cancel context.CancelCauseFunc
+ }
+)
+
+// New builds an engine whose monitors live until appCtx ends or Stop is called.
+func New(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities Activities) (*Engine, error) {
+ if err := runtime.RegisterActor(evidenceActorType, func(string, *actor.Service) actor.Actor { return struct{}{} }); err != nil {
+ return nil, err
+ }
+ ctx, cancel := context.WithCancel(appCtx)
+ return &Engine{
+ runtime: runtime,
+ coordinator: coordinator,
+ activities: activities,
+ ctx: ctx,
+ cancel: cancel,
+ monitors: map[string]*instanceMonitor{},
+ roots: map[string]rootCancellation{},
+ tracked: map[string]context.Context{},
+ admitted: map[string]struct{}{},
+ ready: make(chan struct{}),
+ submitted: map[string]struct{}{},
+ }, nil
+}
+
+// Define validates and registers a workflow. It must run before the host starts.
+func (e *Engine) Define(def Definition) (*Workflow, error) {
+ switch {
+ case def.Version < 1:
+ return nil, fmt.Errorf("workflow %q requires a version", def.Name)
+ case def.Concurrency < 1:
+ return nil, fmt.Errorf("workflow %q requires a concurrency", def.Name)
+ case def.Timeout <= 0:
+ return nil, fmt.Errorf("workflow %q requires a timeout", def.Name)
+ case len(def.Steps) == 0:
+ return nil, fmt.Errorf("workflow %q requires steps", def.Name)
+ }
+ options := append([]workflow.Option{
+ workflow.WithVersion(def.Version),
+ workflow.WithConcurrency(def.Concurrency),
+ workflow.WithTimeout(def.Timeout),
+ workflow.WithSteps(def.Steps...),
+ workflow.WithUnknownVersionPolicy(workflow.FailUnknownVersion),
+ workflow.WithMaxInputSize(defaultMaxPayloadSize),
+ workflow.WithMaxOutputSize(defaultMaxPayloadSize),
+ workflow.WithMaxJournalSize(defaultMaxJournalSize),
+ workflow.WithLogger(slog.Default().With("scope", "workflow")),
+ }, def.Options...)
+ wf, err := workflow.New(def.Name, options...)
+ if err != nil {
+ return nil, fmt.Errorf("define workflow %q: %w", def.Name, err)
+ }
+ if registerErr := e.runtime.RegisterWorkflow(wf); registerErr != nil {
+ return nil, registerErr
+ }
+ defined := &Workflow{def: def, wf: wf, svc: wf.Service(e.runtime.Service())}
+ e.mu.Lock()
+ e.workflows = append(e.workflows, defined)
+ e.mu.Unlock()
+ return defined, nil
+}
+
+// Fingerprint is the graph hash the definition declares for its version.
+func (w *Workflow) Fingerprint() string { return w.def.Fingerprint }
+
+// Francis exposes the underlying definition for use with workflow.WithChild.
+func (w *Workflow) Francis() *workflow.Workflow { return w.wf }
+
+// Workflows lists every defined workflow.
+func (e *Engine) Workflows() []*Workflow {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ return slices.Clone(e.workflows)
+}
+
+// Start monitors every unfinished root instance so activities complete without a caller.
+func (e *Engine) Start(ctx context.Context) error {
+ return e.eachRoot(ctx, func(wf *Workflow, instanceID string) error {
+ e.monitor(wf, instanceID)
+ return nil
+ })
+}
+
+// Ready lets standalone tasks run once startup recovery has protected the work Standalone reported.
+func (e *Engine) Ready() {
+ e.readyOnce.Do(func() { close(e.ready) })
+}
+
+// Standalone lists unfinished instances started by Submit or an unbound Run as runs of the workflow's
+// name, carrying the targets their tasks recorded, so startup recovery can protect their work.
+func (e *Engine) Standalone(ctx context.Context) ([]scheduler.Run, error) {
+ var standalone []scheduler.Run
+ err := e.eachRoot(ctx, func(wf *Workflow, instanceID string) error {
+ if isInstanceOf(instanceID, wf.def.Name) {
+ return nil
+ }
+ run := scheduler.Run{ID: instanceID, JobID: wf.def.Name, ActivityID: wf.activityFor(instanceID), Status: scheduler.Running}
+ if err := e.runtime.Service().GetState(ctx, evidenceActorType, instanceID, &run.Outcome.Targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return fmt.Errorf("load %s progress: %w", instanceID, err)
+ }
+ standalone = append(standalone, run)
+ return nil
+ })
+ return standalone, err
+}
+
+// eachRoot visits every unfinished root instance of every defined workflow.
+func (e *Engine) eachRoot(ctx context.Context, visit func(wf *Workflow, instanceID string) error) error {
+ for _, wf := range e.Workflows() {
+ for _, status := range []workflow.Status{workflow.StatusPending, workflow.StatusRunning, workflow.StatusSuspended, workflow.StatusCompensating} {
+ after := ""
+ for {
+ page, err := wf.svc.List(ctx, &workflow.ListOptions{Status: status, After: after, Limit: 100})
+ if err != nil {
+ return fmt.Errorf("list %s workflows: %w", wf.def.Name, err)
+ }
+ for _, instance := range page.Instances {
+ if instance.Parent != nil {
+ continue
+ }
+ if visitErr := visit(wf, instance.InstanceID); visitErr != nil {
+ return visitErr
+ }
+ }
+ if after = page.AfterID(); after == "" {
+ break
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// Stop ends monitoring; unfinished instances resume on the next Start.
+func (e *Engine) Stop(context.Context) error {
+ e.cancel()
+ return nil
+}
+
+// Run starts the workflow, or attaches to the run's active instance, and waits for its outcome. A run in
+// ctx (jobcontext.Run) binds the instance to that run; set fields of activity override the definition's
+// activity template; a non-nil output receives the last step's data alongside its scheduler.Outcome fields.
+func (e *Engine) Run(ctx context.Context, wf *Workflow, payload any, activity activitylib.StartRequest, output any) (scheduler.Outcome, error) {
+ run, bound := jobcontext.Run(ctx)
+ instanceID := ""
+ if bound {
+ defer e.admit(run.ID)()
+ if target, ok := activeTarget(run, wf.def.Name, false); ok {
+ status, err := wf.svc.GetStatus(ctx, target.ID)
+ if err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) {
+ return scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)
+ }
+ if err == nil && status.Version == wf.def.Version {
+ instanceID = target.ID
+ } else {
+ e.supersede(ctx, wf, target, "Superseded by a newer workflow instance")
+ }
+ }
+ }
+ if instanceID == "" {
+ var err error
+ if instanceID, err = e.startInstance(ctx, wf, run, bound, payload, activity, nil); err != nil {
+ return scheduler.Outcome{Status: scheduler.Failed, Message: err.Error()}, err
+ }
+ }
+ outcome, err := e.await(ctx, wf, run, bound, instanceID)
+ if err != nil || output == nil {
+ return outcome, err
+ }
+ status, err := wf.svc.GetStatus(ctx, instanceID)
+ if err != nil {
+ return outcome, fmt.Errorf("read %s output: %w", wf.def.Name, err)
+ }
+ return outcome, status.DecodeOutput(output)
+}
+
+// Submit starts a standalone instance without waiting for it and returns its ID,
+// which is also its activity's ID. Its tasks record target progress like a run's;
+// evidence is recorded before the instance starts, such as the record it owns.
+func (e *Engine) Submit(ctx context.Context, wf *Workflow, payload any, activity activitylib.StartRequest, evidence ...scheduler.TargetOutcome) (string, error) {
+ instanceID, err := e.startInstance(ctx, wf, scheduler.Run{}, false, payload, activity, evidence)
+ if err != nil {
+ return "", err
+ }
+ e.monitor(wf, instanceID)
+ return instanceID, nil
+}
+
+// Wait blocks until a submitted instance finishes, ctx ends, or the engine stops.
+func (e *Engine) Wait(ctx context.Context, wf *Workflow, instanceID string) (scheduler.Outcome, error) {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ defer context.AfterFunc(e.ctx, cancel)()
+ return e.await(ctx, wf, scheduler.Run{}, false, instanceID)
+}
+
+// Resume attaches to the run's active instance, or reads the result of one that finished before the run saved it.
+// It returns ErrNotResumable when the instance is gone or belongs to a version this host no longer serves.
+func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (scheduler.Outcome, error) {
+ target, ok := activeTarget(run, wf.def.Name, true)
+ if !ok {
+ return scheduler.Outcome{}, ErrNotResumable
+ }
+ defer e.admit(run.ID)()
+ status, err := wf.svc.GetStatus(ctx, target.ID)
+ if err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) {
+ return scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)
+ }
+ if err != nil || status.Version != wf.def.Version {
+ if !target.Status.Terminal() {
+ e.supersede(ctx, wf, target, "Workflow instance could not be resumed")
+ }
+ return scheduler.Outcome{}, ErrNotResumable
+ }
+ return e.await(ctx, wf, run, true, target.ID)
+}
+
+// Retire cancels the run's unfinished instance without resuming it, such as for a job that was disabled. An instance
+// that finished before the run saved its result reports that result instead, with finished set.
+func (e *Engine) Retire(ctx context.Context, wf *Workflow, run scheduler.Run, reason string) (outcome scheduler.Outcome, finished bool, err error) {
+ if target, ok := activeTarget(run, wf.def.Name, true); ok && target.Status.Terminal() {
+ if outcome, err = e.Resume(ctx, wf, run); !errors.Is(err, ErrNotResumable) {
+ return outcome, true, err
+ }
+ }
+ if target, ok := activeTarget(run, wf.def.Name, false); ok {
+ e.supersede(ctx, wf, target, reason)
+ }
+ return scheduler.Outcome{}, false, nil
+}
+
+func (e *Engine) startInstance(ctx context.Context, wf *Workflow, run scheduler.Run, bound bool, payload any, activity activitylib.StartRequest, evidence []scheduler.TargetOutcome) (string, error) {
+ instanceID := wf.def.Name + "-" + uuid.NewV7().String()
+ binding := Binding{RootID: instanceID}
+ if bound {
+ instanceID = fmt.Sprintf("%s-%s-%d", wf.def.Name, run.ID, run.AttemptCount)
+ binding = Binding{EnvironmentID: run.EnvironmentID, JobID: run.JobID, RunID: run.ID, RootID: instanceID}
+ }
+ encoded, err := json.Marshal(payload)
+ if err != nil {
+ return "", fmt.Errorf("encode %s input: %w", wf.def.Name, err)
+ }
+ activityID := wf.activityFor(instanceID)
+ if bound {
+ // Record the instance first so its tasks pass the run fence.
+ target := scheduler.TargetOutcome{ResourceType: scheduler.WorkflowTarget, ID: instanceID, Status: scheduler.Running, ActivityID: activityID}
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return "", fmt.Errorf("record %s instance: %w", wf.def.Name, progressErr)
+ }
+ }
+ if activityID != "" {
+ if activityErr := e.startActivity(ctx, wf, activityID, cmp.Or(run.EnvironmentID, "0"), activity); activityErr != nil {
+ return "", activityErr
+ }
+ }
+ if !bound {
+ e.mu.Lock()
+ e.submitted[instanceID] = struct{}{}
+ e.mu.Unlock()
+ }
+ startErr := error(nil)
+ if len(evidence) > 0 {
+ startErr = e.runtime.Service().SetState(ctx, evidenceActorType, instanceID, evidence, nil)
+ }
+ if startErr == nil {
+ _, _, startErr = wf.svc.Start(ctx, envelope{Binding: binding, Payload: encoded}, workflow.WithInstanceID(instanceID))
+ }
+ if startErr != nil {
+ e.releaseRoot(instanceID)
+ if len(evidence) > 0 {
+ _ = e.runtime.Service().DeleteState(context.WithoutCancel(ctx), evidenceActorType, instanceID)
+ }
+ if activityID != "" {
+ e.untrack(activityID)
+ message := startErr.Error()
+ _, _ = e.activities.CompleteActivity(ctx, activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)
+ }
+ return "", fmt.Errorf("start %s: %w", wf.def.Name, startErr)
+ }
+ return instanceID, nil
+}
+
+func (e *Engine) startActivity(ctx context.Context, wf *Workflow, activityID, environmentID string, details activitylib.StartRequest) error {
+ request := wf.def.Activity
+ request.ID = activityID
+ request.EnvironmentID = cmp.Or(details.EnvironmentID, request.EnvironmentID, environmentID)
+ request.ResourceType = cmp.Or(details.ResourceType, request.ResourceType)
+ request.ResourceID = cmp.Or(details.ResourceID, request.ResourceID)
+ request.ResourceName = cmp.Or(details.ResourceName, request.ResourceName)
+ request.StartedBy = cmp.Or(details.StartedBy, request.StartedBy)
+ request.Step = cmp.Or(details.Step, request.Step)
+ request.LatestMessage = cmp.Or(details.LatestMessage, request.LatestMessage)
+ if details.Metadata != nil {
+ request.Metadata = details.Metadata
+ }
+ // The registration a user cancel reaches exists before the activity does, so no cancel is missed; the
+ // monitor takes it over once the instance runs.
+ tracked := e.activities.Track(e.ctx, activityID)
+ e.mu.Lock()
+ e.tracked[activityID] = tracked
+ e.mu.Unlock()
+ if _, err := e.activities.StartActivity(ctx, request); err != nil {
+ e.untrack(activityID)
+ // Completing drops the service's cancel registration and settles a row the failed start may have left.
+ message := err.Error()
+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)
+ return fmt.Errorf("start %s activity: %w", wf.def.Name, err)
+ }
+ // A deferred slot is awaited by the task that does the work, so a submit never blocks on it.
+ if request.Queue && !request.DeferSlot {
+ waitCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ defer context.AfterFunc(tracked, cancel)()
+ if err := e.activities.AwaitActivitySlotBounded(waitCtx, activityID, request.EnvironmentID); err != nil && !errors.Is(context.Cause(tracked), activitylib.ErrCanceled) {
+ e.untrack(activityID)
+ message := err.Error()
+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)
+ return err
+ }
+ }
+ if errors.Is(context.Cause(tracked), activitylib.ErrCanceled) {
+ e.untrack(activityID)
+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusCancelled, activitylib.ErrCanceled.Error(), nil)
+ return fmt.Errorf("start %s: %w", wf.def.Name, activitylib.ErrCanceled)
+ }
+ return nil
+}
+
+// untrack drops a starting activity's registration; the engine only holds it until the monitor starts.
+func (e *Engine) untrack(activityID string) {
+ e.mu.Lock()
+ delete(e.tracked, activityID)
+ e.mu.Unlock()
+}
+
+// await waits for the instance's outcome, projecting steps onto a bound run at
+// most every stepWriteInterval.
+func (e *Engine) await(ctx context.Context, wf *Workflow, run scheduler.Run, bound bool, instanceID string) (scheduler.Outcome, error) {
+ monitor := e.monitor(wf, instanceID)
+ var recorded []scheduler.StepOutcome
+ var lastWrite time.Time
+ for {
+ steps, outcome, changed := monitor.snapshot()
+ var flush <-chan time.Time
+ if bound && !stepsEqual(recorded, steps) {
+ if wait := stepWriteInterval - time.Since(lastWrite); outcome == nil && wait > 0 {
+ flush = time.After(wait)
+ } else if err := e.coordinator.RecordSteps(ctx, run, instanceID, steps); err != nil && ctx.Err() == nil {
+ slog.DebugContext(ctx, "Failed to record workflow steps", "instanceId", instanceID, "error", err)
+ } else {
+ recorded, lastWrite = steps, time.Now()
+ }
+ }
+ if outcome != nil {
+ if bound {
+ // Bound to the executor: a write lost to shutdown is redone when reconcile re-attaches.
+ target := scheduler.TargetOutcome{ResourceType: scheduler.WorkflowTarget, ID: instanceID, Status: outcome.Status, ActivityID: wf.activityFor(instanceID)}
+ // Run never attaches a finished instance, so a retry starts a new one.
+ if !target.Status.Terminal() {
+ target.Status = scheduler.Failed
+ }
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ slog.WarnContext(ctx, "Failed to record workflow completion", "instanceId", instanceID, "error", err)
+ }
+ }
+ return *outcome, nil
+ }
+ select {
+ case <-ctx.Done():
+ return scheduler.Outcome{}, ctx.Err()
+ case <-changed:
+ case <-flush:
+ }
+ }
+}
+
+// supersede retires an instance that a run no longer follows.
+func (e *Engine) supersede(ctx context.Context, wf *Workflow, target scheduler.TargetOutcome, reason string) {
+ target.Status = scheduler.Failed
+ target.Message = reason
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ slog.WarnContext(ctx, "Failed to retire workflow instance", "instanceId", target.ID, "error", err)
+ }
+ if err := wf.svc.Cancel(ctx, target.ID, reason); err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) && !errors.Is(err, workflow.ErrInstanceTerminated) {
+ slog.DebugContext(ctx, "Failed to cancel superseded workflow instance", "instanceId", target.ID, "error", err)
+ }
+ e.mu.Lock()
+ monitor := e.monitors[target.ID]
+ delete(e.monitors, target.ID)
+ e.mu.Unlock()
+ if monitor != nil {
+ monitor.stop()
+ }
+ e.releaseRoot(target.ID)
+ if target.ActivityID != "" {
+ _, _ = e.activities.CompleteActivity(ctx, target.ActivityID, activitytypes.StatusFailed, reason, new(reason))
+ }
+}
+
+// admit lets the run's tasks execute until the returned func is called.
+func (e *Engine) admit(runID string) func() {
+ e.mu.Lock()
+ e.admitted[runID] = struct{}{}
+ e.mu.Unlock()
+ return func() {
+ e.mu.Lock()
+ delete(e.admitted, runID)
+ e.mu.Unlock()
+ }
+}
+
+// rootContext is canceled, with the user's cause, when a root instance is canceled.
+func (e *Engine) rootContext(rootID string) context.Context {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ root, ok := e.roots[rootID]
+ if !ok {
+ ctx, cancel := context.WithCancelCause(e.ctx)
+ root = rootCancellation{ctx: ctx, cancel: cancel}
+ e.roots[rootID] = root
+ }
+ return root.ctx
+}
+
+// cancelRoot interrupts the running tasks of a root instance and its children.
+func (e *Engine) cancelRoot(rootID string, cause error) {
+ e.mu.Lock()
+ root, ok := e.roots[rootID]
+ e.mu.Unlock()
+ if ok {
+ root.cancel(cause)
+ }
+}
+
+func (e *Engine) releaseRoot(rootID string) {
+ e.mu.Lock()
+ root, ok := e.roots[rootID]
+ delete(e.roots, rootID)
+ delete(e.submitted, rootID)
+ e.mu.Unlock()
+ if ok {
+ root.cancel(errInstanceFinished)
+ }
+}
+
+// activeTarget returns the run's latest instance of a workflow and whether the run may attach to it: an unfinished
+// instance, or when recovering, a finished one whose attempt ended before the run saved its result.
+func activeTarget(run scheduler.Run, name string, recovering bool) (scheduler.TargetOutcome, bool) {
+ for _, target := range slices.Backward(run.Outcome.Targets) {
+ if target.ResourceType != scheduler.WorkflowTarget || !isInstanceOf(target.ID, name) {
+ continue
+ }
+ if !recovering || !target.Status.Terminal() {
+ return target, !target.Status.Terminal()
+ }
+ attempt, err := strconv.Atoi(target.ID[strings.LastIndexByte(target.ID, '-')+1:])
+ if err != nil {
+ return target, false
+ }
+ // Redeliveries repeat an attempt's number; only its latest entry records whether the result was saved.
+ for _, previous := range slices.Backward(run.Attempts) {
+ if previous.Number == attempt {
+ return target, previous.FinishedAt == nil
+ }
+ }
+ return target, false
+ }
+ return scheduler.TargetOutcome{}, false
+}
+
+// isInstanceOf matches run-bound instance IDs of the form <name>-<runID>-<attempt>.
+func isInstanceOf(instanceID, name string) bool {
+ rest, ok := strings.CutPrefix(instanceID, name+"-")
+ if !ok || len(rest) < 38 || rest[36] != '-' {
+ return false
+ }
+ _, err := uuid.Parse(rest[:36])
+ return err == nil
+}
+
+// rootWorkflow returns the workflow a root instance ID belongs to: <name>-<uuid> standalone, or <name>-<runID>-<attempt>.
+func (e *Engine) rootWorkflow(rootID string) *Workflow {
+ for _, wf := range e.Workflows() {
+ rest, ok := strings.CutPrefix(rootID, wf.def.Name+"-")
+ if !ok {
+ continue
+ }
+ if _, err := uuid.Parse(rest); err == nil || isInstanceOf(rootID, wf.def.Name) {
+ return wf
+ }
+ }
+ return nil
+}
+
+// activityFor returns the activity an instance owns, or "" when the workflow has
+// none. The instance ID doubles as the activity ID, so redelivery finds the same row.
+func (w *Workflow) activityFor(instanceID string) string {
+ if w.def.Activity.Type == "" {
+ return ""
+ }
+ return instanceID
+}
+
+// stepLabel names a step for the activity.
+func (w *Workflow) stepLabel(name string) string {
+ return cmp.Or(w.def.Labels[name], name)
+}
diff --git a/backend/pkg/scheduler/flow/flow_test.go b/backend/pkg/scheduler/flow/flow_test.go
new file mode 100644
index 000000000..7feacdbf8
--- /dev/null
+++ b/backend/pkg/scheduler/flow/flow_test.go
@@ -0,0 +1,442 @@
+package flow_test
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/builtin/workflow"
+ "github.com/stretchr/testify/require"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+)
+
+type activitiesFake struct {
+ mu sync.Mutex
+ statuses map[string]activitytypes.Status
+ completed map[string]int
+ tracked map[string]context.CancelCauseFunc
+}
+
+type hostFixture struct {
+ harness *flowtest.Harness
+ job *flow.Job
+ cancel context.CancelFunc
+}
+
+func TestRestartBetweenStepsReattachesSQLite(t *testing.T) {
+ testRestartBetweenStepsInternal(t, "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db")))
+}
+
+func TestRestartBetweenStepsReattachesPostgres(t *testing.T) {
+ databaseURL := os.Getenv("ARCANE_TEST_POSTGRES_DSN")
+ if databaseURL == "" {
+ t.Skip("ARCANE_TEST_POSTGRES_DSN is not set")
+ }
+ testRestartBetweenStepsInternal(t, databaseURL)
+}
+
+func testRestartBetweenStepsInternal(t *testing.T, databaseURL string) {
+ jobID := fmt.Sprintf("flow-restart-%d", time.Now().UnixNano())
+ activities := newActivitiesFake()
+ var firstRuns atomic.Int32
+ blocked := make(chan struct{})
+ first := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(ctx context.Context, _ flow.Task) (any, error) {
+ close(blocked)
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }, nil)
+ run, err := first.harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})
+ require.NoError(t, err)
+ select {
+ case <-blocked:
+ case <-time.After(20 * time.Second):
+ t.Fatal("second step never started")
+ }
+ started, err := first.harness.Coordinator.Get(t.Context(), "0", jobID, run.ID)
+ require.NoError(t, err)
+ instanceID := workflowTargetInternal(t, started).ID
+ first.stop(t)
+
+ second := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(context.Context, flow.Task) (any, error) {
+ return nil, nil
+ }, nil)
+ finished := waitTerminalInternal(t, second, jobID, run.ID)
+
+ require.Equal(t, scheduler.Succeeded, finished.Status)
+ require.Equal(t, int32(1), firstRuns.Load(), "completed steps must not run again")
+ target := workflowTargetInternal(t, finished)
+ require.Equal(t, instanceID, target.ID, "reconcile must re-attach to the same instance")
+ require.Equal(t, scheduler.Succeeded, target.Status)
+ require.Len(t, finished.Outcome.Steps, 3)
+ for _, step := range finished.Outcome.Steps {
+ require.Equal(t, scheduler.Succeeded, step.Status, step.Name)
+ }
+ require.Eventually(t, func() bool { return activities.status(instanceID) == activitytypes.StatusSuccess }, 10*time.Second, 50*time.Millisecond)
+}
+
+func TestVersionBumpFallsBack(t *testing.T) {
+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
+ jobID := fmt.Sprintf("flow-version-%d", time.Now().UnixNano())
+ activities := newActivitiesFake()
+ var firstRuns atomic.Int32
+ blocked := make(chan struct{})
+ first := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(ctx context.Context, _ flow.Task) (any, error) {
+ close(blocked)
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }, nil)
+ run, err := first.harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})
+ require.NoError(t, err)
+ select {
+ case <-blocked:
+ case <-time.After(20 * time.Second):
+ t.Fatal("second step never started")
+ }
+ first.stop(t)
+
+ var fallbacks atomic.Int32
+ second := startHostInternal(t, databaseURL, jobID, 2, activities, &firstRuns, func(context.Context, flow.Task) (any, error) {
+ return nil, nil
+ }, func(context.Context, scheduler.Run) (scheduler.Outcome, error) {
+ fallbacks.Add(1)
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "recovered"}, nil
+ })
+ finished := waitTerminalInternal(t, second, jobID, run.ID)
+
+ require.Equal(t, scheduler.Succeeded, finished.Status)
+ require.Equal(t, "recovered", finished.Outcome.Message)
+ require.Equal(t, int32(1), fallbacks.Load())
+ target := workflowTargetInternal(t, finished)
+ require.Equal(t, scheduler.Failed, target.Status, "the unserved instance must be retired")
+}
+
+func TestChildInheritsRunBinding(t *testing.T) {
+ jobID := fmt.Sprintf("flow-child-%d", time.Now().UnixNano())
+ harness := flowtest.New(t, newActivitiesFake())
+ engine := harness.Engine
+ childRuns := make(chan string, 1)
+ child, err := engine.Define(flow.Definition{
+ Name: "flow-child", Version: 1, Concurrency: 1, Timeout: time.Minute,
+ Steps: []workflow.StepSpec{
+ workflow.Step("work", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ var label string
+ if err := t.Payload(&label); err != nil {
+ return nil, err
+ }
+ run, _ := jobcontext.Run(ctx)
+ childRuns <- run.ID
+ return scheduler.Outcome{Status: scheduler.Partial, Message: "child " + label}, nil
+ })),
+ },
+ })
+ require.NoError(t, err)
+ parent, err := engine.Define(flow.Definition{
+ Name: "flow-parent", Version: 1, Concurrency: 1, Timeout: time.Minute,
+ Steps: []workflow.StepSpec{
+ workflow.Step("prepare", engine.Handler(func(_ context.Context, t flow.Task) (any, error) {
+ return t.ChildInput("payload")
+ })),
+ workflow.Child("child", workflow.WithDefinition(child.Francis())),
+ workflow.Step("finalize", engine.Handler(func(_ context.Context, t flow.Task) (any, error) {
+ var childOutcome scheduler.Outcome
+ if decodeErr := t.DecodeOutput("child", &childOutcome); decodeErr != nil {
+ return nil, decodeErr
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: childOutcome.Message}, nil
+ })),
+ },
+ })
+ require.NoError(t, err)
+ fixture := startJobInternal(t, harness, &flow.Job{Engine: engine, Workflow: parent, JobName: jobID, ScheduleFn: func(context.Context) string { return "" }})
+
+ run, err := harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})
+ require.NoError(t, err)
+ finished := waitTerminalInternal(t, fixture, jobID, run.ID)
+
+ require.Equal(t, scheduler.Succeeded, finished.Status)
+ require.Equal(t, "child payload", finished.Outcome.Message)
+ require.Equal(t, run.ID, <-childRuns, "child tasks must run inside the parent's run")
+}
+
+func TestActivityCancelInterruptsRunningStep(t *testing.T) {
+ activities := newActivitiesFake()
+ harness := flowtest.New(t, activities)
+ engine := harness.Engine
+ started := make(chan string, 1)
+ causes := make(chan error, 1)
+ wf, err := engine.Define(flow.Definition{
+ Name: "flow-cancel", Version: 1, Concurrency: 1, Timeout: time.Minute,
+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},
+ Steps: []workflow.StepSpec{
+ workflow.Step("block", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
+ started <- t.ActivityID()
+ <-ctx.Done()
+ causes <- context.Cause(ctx)
+ return nil, ctx.Err()
+ })),
+ },
+ })
+ require.NoError(t, err)
+ harness.Start(t)
+
+ outcomes := make(chan scheduler.Outcome, 1)
+ go func() {
+ outcome, _ := engine.Run(t.Context(), wf, nil, activitylib.StartRequest{}, nil)
+ outcomes <- outcome
+ }()
+ activityID := <-started
+ activities.requestCancel(activityID)
+
+ select {
+ case cause := <-causes:
+ require.ErrorIs(t, cause, activitylib.ErrCanceled, "the running step must stop when the user cancels")
+ case <-time.After(10 * time.Second):
+ t.Fatal("the running step was not interrupted")
+ }
+ select {
+ case outcome := <-outcomes:
+ require.Equal(t, scheduler.Canceled, outcome.Status)
+ case <-time.After(20 * time.Second):
+ t.Fatal("the workflow did not finish after cancel")
+ }
+ require.Equal(t, activitytypes.StatusCancelled, activities.status(activityID))
+}
+
+func TestStartFinishesSubmittedInstanceWithItsProgress(t *testing.T) {
+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
+ activities := newActivitiesFake()
+ started := make(chan struct{})
+ define := func(harness *flowtest.Harness, work func(context.Context, flow.Task) (any, error)) *flow.Workflow {
+ wf, err := harness.Engine.Define(flow.Definition{
+ Name: "flow-orphan", Version: 1, Concurrency: 1, Timeout: time.Minute,
+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},
+ Steps: []workflow.StepSpec{workflow.Step("work", harness.Engine.Handler(work))},
+ })
+ require.NoError(t, err)
+ return wf
+ }
+
+ first := flowtest.New(t, activities, databaseURL)
+ wf := define(first, func(ctx context.Context, _ flow.Task) (any, error) {
+ if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "backup", Status: scheduler.Running, RecoveryData: []byte(`{"backupId":"frozen"}`)}); err != nil {
+ return nil, err
+ }
+ close(started)
+ <-ctx.Done()
+ return nil, ctx.Err()
+ })
+ first.Start(t)
+ activityID, err := first.Engine.Submit(t.Context(), wf, nil, activitylib.StartRequest{})
+ require.NoError(t, err)
+ <-started
+ standalone, err := first.Engine.Standalone(t.Context())
+ require.NoError(t, err)
+ require.Len(t, standalone, 1)
+ require.Equal(t, activityID, standalone[0].ActivityID)
+ require.JSONEq(t, `{"backupId":"frozen"}`, string(standalone[0].Outcome.Targets[0].RecoveryData))
+ (&hostFixture{harness: first, cancel: func() {}}).stop(t)
+
+ // No caller awaits the instance on the new host; Start alone must finish it, and the
+ // redelivered task must see the progress the interrupted one recorded.
+ second := flowtest.New(t, activities, databaseURL)
+ define(second, func(ctx context.Context, _ flow.Task) (any, error) {
+ previous, _ := jobcontext.Run(ctx)
+ if len(previous.Outcome.Targets) != 1 || string(previous.Outcome.Targets[0].RecoveryData) != `{"backupId":"frozen"}` {
+ return nil, errors.New("interrupted progress was lost")
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded}, nil
+ })
+ second.Start(t)
+ require.Eventually(t, func() bool { return activities.status(activityID) == activitytypes.StatusSuccess }, 20*time.Second, 50*time.Millisecond)
+ standalone, err = second.Engine.Standalone(t.Context())
+ require.NoError(t, err)
+ require.Empty(t, standalone)
+}
+
+func TestRetryable(t *testing.T) {
+ var netErr net.Error = &net.DNSError{IsTimeout: true}
+ cases := map[string]struct {
+ err error
+ want bool
+ }{
+ "unavailable": {common.Classify(common.ErrUnavailable, errors.New("503")), true},
+ "timeout": {common.Classify(common.ErrTimeout, errors.New("slow")), true},
+ "deadline": {fmt.Errorf("lookup: %w", context.DeadlineExceeded), true},
+ "network": {netErr, true},
+ "tls": {&url.Error{Op: "Get", URL: "https://registry", Err: errors.New("x509: certificate signed by unknown authority")}, false},
+ "unauthorized": {common.Classify(common.ErrUnauthorized, errors.New("denied")), false},
+ "plain": {errors.New("boom"), false},
+ }
+ for name, tc := range cases {
+ require.Equal(t, tc.want, flow.Retryable(tc.err), name)
+ }
+}
+
+func startHostInternal(
+ t *testing.T,
+ databaseURL, jobID string,
+ version int,
+ activities *activitiesFake,
+ firstRuns *atomic.Int32,
+ second func(context.Context, flow.Task) (any, error),
+ fallback func(context.Context, scheduler.Run) (scheduler.Outcome, error),
+) *hostFixture {
+ t.Helper()
+ harness := flowtest.New(t, activities, databaseURL)
+ engine := harness.Engine
+ steps := []workflow.StepSpec{
+ workflow.Step("first", engine.Handler(func(context.Context, flow.Task) (any, error) {
+ firstRuns.Add(1)
+ return nil, nil
+ })),
+ workflow.Step("second", engine.Handler(second), workflow.WithMaxAttempts(1)),
+ workflow.Step("finalize", engine.Handler(func(context.Context, flow.Task) (any, error) {
+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "done"}, nil
+ })),
+ }
+ if version > 1 {
+ steps = append(steps[:2], workflow.Step("extra", engine.Handler(func(context.Context, flow.Task) (any, error) { return nil, nil })), steps[2])
+ }
+ wf, err := engine.Define(flow.Definition{
+ Name: "flow-test",
+ Version: version,
+ Concurrency: 2,
+ Timeout: time.Minute,
+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},
+ Steps: steps,
+ })
+ require.NoError(t, err)
+ job := &flow.Job{Engine: engine, Workflow: wf, JobName: jobID, ScheduleFn: func(context.Context) string { return "" }, FallbackFn: fallback}
+ return startJobInternal(t, harness, job)
+}
+
+// startJobInternal starts the host and a coordinator that executes job.
+func startJobInternal(t *testing.T, harness *flowtest.Harness, job *flow.Job) *hostFixture {
+ t.Helper()
+ coordinator := harness.Coordinator
+ coordinator.SetExecutor(
+ func(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {
+ return job.Run(coordinator.ExecutionContext(ctx, run, ""))
+ },
+ func(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {
+ return job.Reconcile(coordinator.ExecutionContext(ctx, run, ""), run)
+ },
+ )
+ harness.Start(t)
+ appCtx, cancel := context.WithCancel(t.Context())
+ require.NoError(t, coordinator.Start(t.Context(), appCtx))
+ coordinator.Activate()
+ fixture := &hostFixture{harness: harness, job: job, cancel: cancel}
+ t.Cleanup(func() { fixture.stop(t) })
+ return fixture
+}
+
+// stop simulates a process exit between steps.
+func (f *hostFixture) stop(t *testing.T) {
+ t.Helper()
+ if f.cancel == nil {
+ return
+ }
+ f.cancel()
+ f.cancel = nil
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 15*time.Second)
+ defer cancel()
+ _ = f.harness.Coordinator.Stop(ctx)
+ _ = f.harness.Engine.Stop(ctx)
+ require.NoError(t, f.harness.Runtime.Stop(ctx))
+}
+
+func waitTerminalInternal(t *testing.T, fixture *hostFixture, jobID, runID string) scheduler.Run {
+ t.Helper()
+ var run scheduler.Run
+ require.Eventually(t, func() bool {
+ var err error
+ run, err = fixture.harness.Coordinator.Get(t.Context(), "0", jobID, runID)
+ return err == nil && run.Status.Terminal()
+ }, 45*time.Second, 100*time.Millisecond)
+ return run
+}
+
+func workflowTargetInternal(t *testing.T, run scheduler.Run) scheduler.TargetOutcome {
+ t.Helper()
+ var found []scheduler.TargetOutcome
+ for _, target := range run.Outcome.Targets {
+ if target.ResourceType == scheduler.WorkflowTarget {
+ found = append(found, target)
+ }
+ }
+ require.Len(t, found, 1)
+ return found[0]
+}
+
+func newActivitiesFake() *activitiesFake {
+ return &activitiesFake{statuses: map[string]activitytypes.Status{}, completed: map[string]int{}, tracked: map[string]context.CancelCauseFunc{}}
+}
+
+func (f *activitiesFake) status(id string) activitytypes.Status {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.statuses[id]
+}
+
+func (f *activitiesFake) StartActivity(_ context.Context, req activitylib.StartRequest) (*activitytypes.Activity, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if _, ok := f.statuses[req.ID]; !ok {
+ f.statuses[req.ID] = activitytypes.StatusRunning
+ }
+ return &activitytypes.Activity{ID: req.ID, Status: f.statuses[req.ID]}, nil
+}
+
+func (f *activitiesFake) CompleteActivity(_ context.Context, activityID string, status activitytypes.Status, _ string, _ *string, _ ...string) (*activitytypes.Activity, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.statuses[activityID] = status
+ f.completed[activityID]++
+ return &activitytypes.Activity{ID: activityID, Status: status}, nil
+}
+
+func (f *activitiesFake) AppendMessage(context.Context, string, activitylib.AppendMessageRequest) (*activitytypes.Message, error) {
+ return &activitytypes.Message{}, nil
+}
+
+func (f *activitiesFake) AppendMessages(context.Context, string, []activitylib.AppendMessageRequest) ([]activitytypes.Message, error) {
+ return nil, nil
+}
+
+func (f *activitiesFake) Track(ctx context.Context, activityID string) context.Context {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ tracked, cancel := context.WithCancelCause(ctx)
+ f.tracked[activityID] = cancel
+ return tracked
+}
+
+// requestCancel cancels an activity the way the activity service does for a user.
+func (f *activitiesFake) requestCancel(activityID string) {
+ f.mu.Lock()
+ cancel := f.tracked[activityID]
+ f.mu.Unlock()
+ cancel(activitylib.ErrCanceled)
+}
+
+func (f *activitiesFake) UpdateActivity(_ context.Context, activityID string, _ activitylib.UpdateRequest) (*activitytypes.Activity, error) {
+ return &activitytypes.Activity{ID: activityID}, nil
+}
+
+func (f *activitiesFake) AwaitActivitySlotBounded(context.Context, string, string) error { return nil }
diff --git a/backend/pkg/scheduler/flow/flowtest/flowtest.go b/backend/pkg/scheduler/flow/flowtest/flowtest.go
new file mode 100644
index 000000000..40f950dec
--- /dev/null
+++ b/backend/pkg/scheduler/flow/flowtest/flowtest.go
@@ -0,0 +1,91 @@
+// Package flowtest runs workflow engines on isolated Francis hosts for tests.
+package flowtest
+
+import (
+ "context"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/italypaleale/francis/builtin/workflow"
+ "github.com/libtnb/sqlite"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
+)
+
+// Harness is an engine with its host and coordinator, ready for workflows to be defined.
+type Harness struct {
+ Runtime *francis.Runtime
+ Coordinator *runs.Coordinator
+ Engine *flow.Engine
+}
+
+// New builds a harness on an in-memory host, or on the given database URL.
+func New(t testing.TB, activities flow.Activities, databaseURLs ...string) *Harness {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ if migrateErr := db.AutoMigrate(&kv.KVEntry{}); migrateErr != nil {
+ t.Fatal(migrateErr)
+ }
+ runtime := francistest.New(t, databaseURLs...)
+ coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)
+ if registerErr := coordinator.Register(runtime); registerErr != nil {
+ t.Fatal(registerErr)
+ }
+ engine, err := flow.New(t.Context(), runtime, coordinator, activities)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &Harness{Runtime: runtime, Coordinator: coordinator, Engine: engine}
+}
+
+// Start starts the host and the engine once every workflow is defined.
+func (h *Harness) Start(t testing.TB) {
+ t.Helper()
+ francistest.Start(t, h.Runtime)
+ if err := h.Engine.Start(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ h.Engine.Ready()
+ t.Cleanup(func() { _ = h.Engine.Stop(context.WithoutCancel(t.Context())) })
+}
+
+// AssertDefinitions fails when a workflow's graph no longer matches the
+// Fingerprint its definition declares, which means it changed without a version bump.
+func AssertDefinitions(t testing.TB, h *Harness) {
+ t.Helper()
+ for _, wf := range h.Engine.Workflows() {
+ definition := wf.Francis()
+ service := definition.Service(h.Runtime.Service())
+ // A fresh host has no instances, so resetting installs this binary's fingerprint.
+ if err := service.ForgetVersion(t.Context(), definition.Version()); err != nil {
+ t.Fatal(err)
+ }
+ infos, err := service.Definitions(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ index := slices.IndexFunc(infos, func(info workflow.DefinitionInfo) bool { return info.Version == definition.Version() })
+ if index < 0 {
+ t.Fatalf("workflow %q version %d is not registered", definition.Name(), definition.Version())
+ }
+ if got := infos[index].Fingerprint; got != wf.Fingerprint() {
+ t.Errorf("workflow %q no longer matches its Fingerprint: bump Version if this change is not yet released, then set Fingerprint to %q", definition.Name(), got)
+ }
+ }
+}
diff --git a/backend/pkg/scheduler/flow/job.go b/backend/pkg/scheduler/flow/job.go
new file mode 100644
index 000000000..2b4d5743e
--- /dev/null
+++ b/backend/pkg/scheduler/flow/job.go
@@ -0,0 +1,76 @@
+package flow
+
+import (
+ "context"
+ "errors"
+
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+)
+
+// Job runs a workflow as a scheduler job. The coordinator's single flight per
+// job and environment already prevents overlapping runs.
+type Job struct {
+ Engine *Engine
+ Workflow *Workflow
+ JobName string
+ // Payload is every instance's input, such as the entity a per-entity job runs for.
+ Payload any
+ // Activity overrides the definition's activity template, such as with the entity's name.
+ Activity activitylib.StartRequest
+ ScheduleFn func(ctx context.Context) string
+ ShouldRunFn func(ctx context.Context) bool
+ // FallbackFn recovers a run whose instance cannot resume; nil fails the run so
+ // a retry starts a fresh instance under a new attempt.
+ FallbackFn func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error)
+ ValidateRetryFn func(ctx context.Context, run scheduler.Run) error
+}
+
+// Name is the scheduler job name.
+func (j *Job) Name() string { return j.JobName }
+
+// Schedule is the job's current cron expression.
+func (j *Job) Schedule(ctx context.Context) string { return j.ScheduleFn(ctx) }
+
+// ShouldSchedule reports whether the job runs; a nil ShouldRunFn always does.
+func (j *Job) ShouldSchedule(ctx context.Context) bool {
+ return j.ShouldRunFn == nil || j.ShouldRunFn(ctx)
+}
+
+// Run skips when the job's condition no longer holds, then runs the workflow.
+func (j *Job) Run(ctx context.Context) (scheduler.Outcome, error) {
+ if !j.ShouldSchedule(ctx) {
+ return scheduler.Outcome{Status: scheduler.Skipped}, nil
+ }
+ return j.Engine.Run(ctx, j.Workflow, j.Payload, j.Activity, nil)
+}
+
+// Reconcile resumes an interrupted run's instance, falling back to FallbackFn when it cannot be resumed.
+// A job disabled since the run started makes no further changes.
+func (j *Job) Reconcile(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {
+ if !j.ShouldSchedule(ctx) {
+ const message = "Job is disabled; the interrupted run was not resumed"
+ if outcome, finished, err := j.Engine.Retire(ctx, j.Workflow, previous, message); finished {
+ return outcome, err
+ }
+ return scheduler.Outcome{Status: scheduler.Failed, Message: message}, nil
+ }
+ outcome, err := j.Engine.Resume(ctx, j.Workflow, previous)
+ if !errors.Is(err, ErrNotResumable) {
+ return outcome, err
+ }
+ if j.FallbackFn != nil {
+ return j.FallbackFn(ctx, previous)
+ }
+ // Running again here could reuse the superseded instance's ID within this attempt.
+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance could not be resumed; retry the run"}, nil
+}
+
+// ValidateRetry rejects a retry that ValidateRetryFn reports unsafe.
+func (j *Job) ValidateRetry(ctx context.Context, run scheduler.Run) error {
+ if j.ValidateRetryFn == nil {
+ return nil
+ }
+ return j.ValidateRetryFn(ctx, run)
+}
diff --git a/backend/pkg/scheduler/flow/monitor.go b/backend/pkg/scheduler/flow/monitor.go
new file mode 100644
index 000000000..8010d759a
--- /dev/null
+++ b/backend/pkg/scheduler/flow/monitor.go
@@ -0,0 +1,337 @@
+package flow
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "slices"
+ "sync"
+ "time"
+
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/builtin/workflow"
+
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+)
+
+const (
+ minPollInterval = 100 * time.Millisecond
+ maxPollInterval = time.Second
+)
+
+type (
+ // instanceMonitor is the single poller for one root instance. It owns the
+ // instance's activity and broadcasts step changes and the final outcome.
+ instanceMonitor struct {
+ mu sync.Mutex
+ steps []scheduler.StepOutcome
+ outcome *scheduler.Outcome
+ changed chan struct{}
+ cancel context.CancelFunc
+ // children caches the workflow serving each child instance.
+ children map[string]*Workflow
+ }
+
+ // activityProjection remembers what was last written to the activity.
+ activityProjection struct {
+ progress int
+ step string
+ statuses map[string]scheduler.RunStatus
+ }
+)
+
+func (e *Engine) monitor(wf *Workflow, instanceID string) *instanceMonitor {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ if monitor := e.monitors[instanceID]; monitor != nil {
+ return monitor
+ }
+ ctx, cancel := context.WithCancel(e.ctx)
+ monitor := &instanceMonitor{changed: make(chan struct{}), cancel: cancel, children: map[string]*Workflow{}}
+ e.monitors[instanceID] = monitor
+ go e.poll(ctx, wf, instanceID, monitor)
+ return monitor
+}
+
+func (e *Engine) poll(ctx context.Context, wf *Workflow, instanceID string, monitor *instanceMonitor) {
+ activityID := wf.activityFor(instanceID)
+ var canceled <-chan struct{}
+ trackedCtx := ctx
+ if activityID != "" {
+ // A starting instance's activity is already tracked; one found on Start is tracked here.
+ e.mu.Lock()
+ trackedCtx = e.tracked[activityID]
+ delete(e.tracked, activityID)
+ e.mu.Unlock()
+ if trackedCtx == nil {
+ trackedCtx = e.activities.Track(ctx, activityID)
+ }
+ canceled = trackedCtx.Done()
+ }
+ userCanceled := false
+ projection := activityProjection{progress: -1, statuses: map[string]scheduler.RunStatus{}}
+ delay := minPollInterval
+ for {
+ status, err := wf.svc.GetStatus(ctx, instanceID)
+ switch {
+ case err == nil:
+ steps := e.projectSteps(ctx, monitor, status)
+ if status.Status.IsTerminal() {
+ outcome := decodeOutcome(status, steps, activityID)
+ // The interrupted task may fail the instance before the cancel turn lands.
+ if userCanceled && outcome.Status == scheduler.Failed {
+ outcome.Status, outcome.Message = scheduler.Canceled, activitylib.ErrCanceled.Error()
+ }
+ e.finish(ctx, wf, instanceID, monitor, outcome)
+ return
+ }
+ if monitor.publish(steps, nil) {
+ e.projectActivity(ctx, wf, activityID, &projection, steps)
+ delay = minPollInterval
+ }
+ case errors.Is(err, workflow.ErrInstanceNotFound):
+ e.finish(ctx, wf, instanceID, monitor, scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance no longer exists", ActivityID: activityID})
+ return
+ case ctx.Err() == nil:
+ slog.DebugContext(ctx, "Failed to read workflow status", "instanceId", instanceID, "error", err)
+ }
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return
+ case <-canceled:
+ canceled = nil
+ if errors.Is(context.Cause(trackedCtx), activitylib.ErrCanceled) {
+ userCanceled = true
+ if cancelErr := wf.svc.Cancel(ctx, instanceID, activitylib.ErrCanceled.Error()); cancelErr != nil {
+ slog.WarnContext(ctx, "Failed to cancel workflow", "instanceId", instanceID, "error", cancelErr)
+ }
+ e.cancelRoot(instanceID, activitylib.ErrCanceled)
+ }
+ case <-timer.C:
+ }
+ timer.Stop()
+ delay = min(delay*2, maxPollInterval)
+ }
+}
+
+func (e *Engine) finish(ctx context.Context, wf *Workflow, instanceID string, monitor *instanceMonitor, outcome scheduler.Outcome) {
+ e.mu.Lock()
+ if e.monitors[instanceID] == monitor {
+ delete(e.monitors, instanceID)
+ }
+ e.mu.Unlock()
+ e.releaseRoot(instanceID)
+ if !isInstanceOf(instanceID, wf.def.Name) {
+ if err := e.runtime.Service().DeleteState(context.WithoutCancel(ctx), evidenceActorType, instanceID); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ slog.WarnContext(ctx, "Failed to delete workflow progress", "instanceId", instanceID, "error", err)
+ }
+ }
+ activityID := wf.activityFor(instanceID)
+ if activityID != "" {
+ status := activitytypes.StatusSuccess
+ var errMessage *string
+ switch outcome.Status {
+ case scheduler.Canceled:
+ status = activitytypes.StatusCancelled
+ case scheduler.Failed, scheduler.Partial, scheduler.NeedsAttention:
+ status = activitytypes.StatusFailed
+ errMessage = new(outcome.Message)
+ case scheduler.Queued, scheduler.Waiting, scheduler.Running, scheduler.Retrying, scheduler.Succeeded, scheduler.Skipped:
+ }
+ if _, err := e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, status, outcome.Message, errMessage); err != nil {
+ slog.WarnContext(ctx, "Failed to complete workflow activity", "instanceId", instanceID, "error", err)
+ }
+ }
+ monitor.publish(outcome.Steps, &outcome)
+ monitor.cancel()
+}
+
+// stop ends a monitor whose instance was superseded.
+func (m *instanceMonitor) stop() {
+ m.cancel()
+ m.publish(nil, &scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance was superseded"})
+}
+
+// publish records a change and wakes waiters; it reports whether anything changed.
+func (m *instanceMonitor) publish(steps []scheduler.StepOutcome, outcome *scheduler.Outcome) bool {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.outcome != nil || (outcome == nil && stepsEqual(m.steps, steps)) {
+ return false
+ }
+ m.steps = steps
+ m.outcome = outcome
+ close(m.changed)
+ m.changed = make(chan struct{})
+ return true
+}
+
+func (m *instanceMonitor) snapshot() ([]scheduler.StepOutcome, *scheduler.Outcome, <-chan struct{}) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.steps, m.outcome, m.changed
+}
+
+// projectSteps maps Francis steps onto run steps; a child step reports its child's fan-out totals.
+func (e *Engine) projectSteps(ctx context.Context, monitor *instanceMonitor, status workflow.InstanceStatus) []scheduler.StepOutcome {
+ steps := make([]scheduler.StepOutcome, 0, len(status.Steps))
+ for _, view := range status.Steps {
+ step := scheduler.StepOutcome{
+ Name: view.Name,
+ Total: view.Tasks,
+ Completed: view.Completed,
+ Failed: view.Failed,
+ Attempts: view.Attempts,
+ Message: view.Error,
+ }
+ if view.Kind == workflow.KindChild && len(view.ChildIDs) == 1 {
+ if child, ok := e.childStatus(ctx, monitor, view.ChildIDs[0]); ok {
+ for _, childStep := range child.Steps {
+ if childStep.Kind == workflow.KindForEach {
+ step.Total, step.Completed, step.Failed = childStep.Tasks, childStep.Completed, childStep.Failed
+ }
+ }
+ }
+ }
+ step.Status = stepStatus(view.Status, step.Failed)
+ if !view.StartedAt.IsZero() {
+ step.StartedAt = new(view.StartedAt)
+ }
+ if !view.CompletedAt.IsZero() {
+ step.FinishedAt = new(view.CompletedAt)
+ }
+ steps = append(steps, step)
+ }
+ return steps
+}
+
+// childStatus reads a child instance, resolving its workflow once per monitor.
+func (e *Engine) childStatus(ctx context.Context, monitor *instanceMonitor, childID string) (workflow.InstanceStatus, bool) {
+ if wf := monitor.children[childID]; wf != nil {
+ status, err := wf.svc.GetStatus(ctx, childID)
+ return status, err == nil
+ }
+ for _, wf := range e.Workflows() {
+ if status, err := wf.svc.GetStatus(ctx, childID); err == nil {
+ monitor.children[childID] = wf
+ return status, true
+ }
+ }
+ return workflow.InstanceStatus{}, false
+}
+
+func stepStatus(status workflow.StepStatus, failed int) scheduler.RunStatus {
+ switch status {
+ case workflow.StepRunning, workflow.StepCompensating:
+ return scheduler.Running
+ case workflow.StepCompleted:
+ if failed > 0 {
+ return scheduler.Partial
+ }
+ return scheduler.Succeeded
+ case workflow.StepFailed, workflow.StepCompensationFailed:
+ return scheduler.Failed
+ case workflow.StepSkipped:
+ return scheduler.Skipped
+ case workflow.StepCompensated:
+ return scheduler.Canceled
+ case workflow.StepPending:
+ return scheduler.Queued
+ }
+ return scheduler.Queued
+}
+
+// decodeOutcome decodes a terminal instance into the run outcome contract.
+func decodeOutcome(status workflow.InstanceStatus, steps []scheduler.StepOutcome, activityID string) scheduler.Outcome {
+ var outcome scheduler.Outcome
+ switch status.Status {
+ case workflow.StatusCompleted:
+ if err := status.DecodeOutput(&outcome); err != nil {
+ outcome = scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow returned an invalid outcome: " + err.Error()}
+ }
+ if outcome.Status == "" {
+ outcome.Status = scheduler.Succeeded
+ }
+ case workflow.StatusCancelled:
+ outcome = scheduler.Outcome{Status: scheduler.Canceled, Message: status.Cause}
+ case workflow.StatusFailed, workflow.StatusPending, workflow.StatusRunning, workflow.StatusSuspended, workflow.StatusCompensating:
+ outcome = scheduler.Outcome{Status: scheduler.Failed, Message: status.Cause}
+ }
+ outcome.Steps = steps
+ if outcome.ActivityID == "" {
+ outcome.ActivityID = activityID
+ }
+ return outcome
+}
+
+// projectActivity writes overall progress and step transitions to the activity.
+func (e *Engine) projectActivity(ctx context.Context, wf *Workflow, activityID string, projection *activityProjection, steps []scheduler.StepOutcome) {
+ if activityID == "" || len(steps) == 0 {
+ return
+ }
+ done := 0.0
+ current := ""
+ for _, step := range steps {
+ previous, seen := projection.statuses[step.Name]
+ if step.Status != scheduler.Queued && (!seen || previous != step.Status) {
+ projection.statuses[step.Name] = step.Status
+ if step.Status != scheduler.Running {
+ e.appendStepMessage(ctx, activityID, wf.stepLabel(step.Name), step)
+ }
+ }
+ switch {
+ case step.Status == scheduler.Running:
+ current = wf.stepLabel(step.Name)
+ if step.Total > 0 {
+ done += float64(step.Completed+step.Failed) / float64(step.Total)
+ }
+ case step.Status != scheduler.Queued:
+ done++
+ }
+ }
+ progress := int(done * 100 / float64(len(steps)))
+ if progress == projection.progress && current == projection.step {
+ return
+ }
+ projection.progress, projection.step = progress, current
+ if _, err := e.activities.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Step: &current}); err != nil {
+ slog.DebugContext(ctx, "Failed to update workflow activity progress", "activityId", activityID, "error", err)
+ }
+}
+
+func (e *Engine) appendStepMessage(ctx context.Context, activityID, label string, step scheduler.StepOutcome) {
+ level := activitytypes.MessageLevelInfo
+ message := fmt.Sprintf("%s: %s", label, step.Status)
+ if step.Total > 1 {
+ message += fmt.Sprintf(" (%d/%d", step.Completed, step.Total)
+ if step.Failed > 0 {
+ message += fmt.Sprintf(", %d failed", step.Failed)
+ }
+ message += ")"
+ }
+ switch step.Status {
+ case scheduler.Failed:
+ level = activitytypes.MessageLevelError
+ if step.Message != "" {
+ message += ": " + step.Message
+ }
+ case scheduler.Partial:
+ level = activitytypes.MessageLevelWarning
+ case scheduler.Queued, scheduler.Waiting, scheduler.Running, scheduler.Retrying, scheduler.Succeeded, scheduler.Skipped, scheduler.NeedsAttention, scheduler.Canceled:
+ }
+ if _, err := e.activities.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{Level: level, Message: message}); err != nil {
+ slog.DebugContext(ctx, "Failed to append workflow step message", "activityId", activityID, "error", err)
+ }
+}
+
+func stepsEqual(left, right []scheduler.StepOutcome) bool {
+ return slices.EqualFunc(left, right, func(a, b scheduler.StepOutcome) bool {
+ return a.Name == b.Name && a.Status == b.Status && a.Total == b.Total && a.Completed == b.Completed &&
+ a.Failed == b.Failed && a.Attempts == b.Attempts && a.Message == b.Message
+ })
+}
diff --git a/backend/pkg/scheduler/flow/task.go b/backend/pkg/scheduler/flow/task.go
new file mode 100644
index 000000000..5e6b8e9bc
--- /dev/null
+++ b/backend/pkg/scheduler/flow/task.go
@@ -0,0 +1,262 @@
+package flow
+
+import (
+ "cmp"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/builtin/workflow"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+)
+
+const (
+ // maxErrorLength bounds task errors recorded in the workflow journal.
+ maxErrorLength = 1024
+ // slotWait bounds how long a resumed task waits for its activity slot before it is handed back.
+ slotWait = 20 * time.Second
+)
+
+type (
+ // Binding ties an instance, and its children, to the run that started it.
+ Binding struct {
+ EnvironmentID string `json:"environmentId,omitempty"`
+ JobID string `json:"jobId,omitempty"`
+ RunID string `json:"runId,omitempty"`
+ RootID string `json:"rootId"`
+ }
+
+ // envelope is every instance's input: its run binding plus the caller's payload.
+ envelope struct {
+ Binding Binding `json:"binding"`
+ Payload json.RawMessage `json:"payload,omitempty"`
+ }
+
+ // Task is what an engine handler receives: the Francis task plus its binding.
+ Task struct {
+ workflow.Task
+ binding Binding
+ payload json.RawMessage
+ // recovered marks a standalone task whose instance an earlier process started.
+ recovered bool
+ }
+
+ // Result is one fan-out slot: the task's output, or the error that failed it.
+ Result[T any] struct {
+ Value T
+ Err string
+ }
+)
+
+// Handler wraps a step handler. Its ctx carries the bound run exactly as a job's
+// Run does, is canceled when the user cancels the root activity, and its errors
+// are classified for Francis retries.
+func (e *Engine) Handler(fn func(ctx context.Context, t Task) (any, error)) workflow.StepOption {
+ return workflow.WithRun(func(ctx context.Context, wt workflow.Task) (output any, err error) {
+ // Francis does not recover handler panics, which would take the process down.
+ defer func() {
+ if panicErr := utils.PanicToError(recover()); panicErr != nil {
+ slog.ErrorContext(ctx, "Workflow task panicked", "workflow", wt.Workflow(), "step", wt.Step(), "error", panicErr)
+ output, err = nil, fmt.Errorf("%w: task panicked: %w", actor.ErrJobPermanentFailure, panicErr)
+ }
+ }()
+ var input envelope
+ if decodeErr := wt.DecodeInput(&input); decodeErr != nil {
+ return nil, fmt.Errorf("%w: decode workflow input: %w", actor.ErrJobPermanentFailure, decodeErr)
+ }
+ // Francis neither interrupts a running job on Cancel nor at host shutdown, so the engine does both.
+ ctx, cancel := context.WithCancelCause(ctx)
+ defer cancel(nil)
+ root := e.rootContext(input.Binding.RootID)
+ defer context.AfterFunc(root, func() { cancel(context.Cause(root)) })()
+ taskCtx, err := e.taskContext(ctx, input.Binding)
+ if err != nil {
+ return nil, classify(ctx, err)
+ }
+ // A resumed instance's activity lost its slot with the old process, so its first task takes it back;
+ // a held slot returns at once. While none frees up, the task is handed back without spending an attempt.
+ if root := e.rootWorkflow(input.Binding.RootID); root != nil && root.def.Activity.Queue && !root.def.Activity.DeferSlot {
+ environmentID := cmp.Or(root.def.Activity.EnvironmentID, input.Binding.EnvironmentID, "0")
+ // The wait stays well inside any attempt timeout, so waiting for a slot never spends an attempt.
+ activityID := root.activityFor(input.Binding.RootID)
+ slotCtx, cancelSlot := context.WithTimeout(taskCtx, slotWait)
+ unlockSlot := e.slots.Lock(activityID)
+ slotErr := e.activities.AwaitActivitySlotBounded(slotCtx, activityID, environmentID)
+ unlockSlot()
+ cancelSlot()
+ if slotErr != nil {
+ if ctx.Err() != nil {
+ return nil, classify(ctx, ctx.Err())
+ }
+ return nil, actor.ErrJobRejected
+ }
+ }
+ e.mu.Lock()
+ _, submitted := e.submitted[input.Binding.RootID]
+ e.mu.Unlock()
+ recovered := input.Binding.RunID == "" && !submitted
+ output, err = fn(taskCtx, Task{Task: wt, binding: input.Binding, payload: input.Payload, recovered: recovered})
+ // An interrupted handler may report a result its interruption caused, so its output is never kept.
+ if err = cmp.Or(err, ctx.Err()); err != nil {
+ return nil, classify(ctx, err)
+ }
+ return output, nil
+ })
+}
+
+// taskContext rebuilds a task's job execution ctx, fencing run-bound tasks.
+func (e *Engine) taskContext(ctx context.Context, binding Binding) (context.Context, error) {
+ if binding.RunID == "" {
+ return e.standaloneContext(ctx, binding)
+ }
+ run, err := e.coordinator.Get(ctx, binding.EnvironmentID, binding.JobID, binding.RunID)
+ if errors.Is(err, runs.ErrRunNotFound) {
+ return nil, fmt.Errorf("%w: run %s no longer exists", actor.ErrJobPermanentFailure, binding.RunID)
+ }
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("load run %s: %w", binding.RunID, err))
+ }
+ if !runs.ActiveWorkflow(run, binding.RootID) {
+ // A run waiting for its next attempt reattaches to this instance, so the instance's tasks wait with it.
+ attempt := run
+ attempt.Status = scheduler.Running
+ if (run.Status == scheduler.Queued || run.Status == scheduler.Waiting || run.Status == scheduler.Retrying) && runs.ActiveWorkflow(attempt, binding.RootID) {
+ return nil, actor.ErrJobRejected
+ }
+ return nil, fmt.Errorf("%w: run %s no longer follows instance %s", actor.ErrJobPermanentFailure, binding.RunID, binding.RootID)
+ }
+ // A task recovered at startup waits until reconcile re-authorizes its run and attaches to it.
+ e.mu.Lock()
+ _, admitted := e.admitted[binding.RunID]
+ e.mu.Unlock()
+ if !admitted {
+ return nil, actor.ErrJobRejected
+ }
+ return e.coordinator.ExecutionContext(ctx, run, binding.RootID), nil
+}
+
+// standaloneContext gives a standalone task the execution a run would give it, with
+// target progress kept in the engine's state for the instance.
+func (e *Engine) standaloneContext(ctx context.Context, binding Binding) (context.Context, error) {
+ select {
+ case <-e.ready:
+ default:
+ // Work started before startup recovery collects protected records could be failed as interrupted.
+ return nil, actor.ErrJobRejected
+ }
+ service := e.runtime.Service()
+ run := scheduler.Run{ID: binding.RootID, EnvironmentID: cmp.Or(binding.EnvironmentID, "0"), Status: scheduler.Running}
+ if err := service.GetState(ctx, evidenceActorType, binding.RootID, &run.Outcome.Targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("load instance %s progress: %w", binding.RootID, err))
+ }
+ ctx = utils.WithActivityBatchID(ctx, binding.RootID)
+ return jobcontext.WithExecution(ctx, run, func(target scheduler.TargetOutcome) error {
+ defer e.evidence.Lock(binding.RootID)()
+ var targets []scheduler.TargetOutcome
+ if err := service.GetState(ctx, evidenceActorType, binding.RootID, &targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return err
+ }
+ return service.SetState(ctx, evidenceActorType, binding.RootID, jobcontext.Merge(targets, target), nil)
+ }), nil
+}
+
+// classify maps handler errors onto Francis delivery semantics.
+func classify(ctx context.Context, err error) error {
+ switch {
+ case errors.Is(err, actor.ErrJobRejected), errors.Is(err, actor.ErrJobPermanentFailure):
+ return err
+ case errors.Is(context.Cause(ctx), activitylib.ErrCanceled):
+ // A user cancel must not be redelivered.
+ return fmt.Errorf("%w: %w", actor.ErrJobPermanentFailure, activitylib.ErrCanceled)
+ case errors.Is(ctx.Err(), context.Canceled):
+ // Host shutdown: redeliver without spending an attempt.
+ return actor.ErrJobRejected
+ case Retryable(err):
+ return errors.New(truncate(err.Error()))
+ default:
+ return fmt.Errorf("%w: %s", actor.ErrJobPermanentFailure, truncate(err.Error()))
+ }
+}
+
+// Retryable reports whether another attempt could succeed. Sources classify
+// transient failures; only network timeouts are assumed transient here.
+func Retryable(err error) bool {
+ var netErr net.Error
+ return errors.Is(err, common.ErrUnavailable) || errors.Is(err, common.ErrTimeout) ||
+ errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout())
+}
+
+// Payload decodes the instance payload.
+func (t Task) Payload(into any) error {
+ if len(t.payload) == 0 {
+ return nil
+ }
+ return json.Unmarshal(t.payload, into)
+}
+
+// Recovered reports a standalone task whose instance an earlier process started, so no caller is waiting on it
+// and the request's own checks did not run in this process.
+func (t Task) Recovered() bool { return t.recovered }
+
+// ActivityID returns the activity owned by the task's root instance.
+func (t Task) ActivityID() string { return t.binding.RootID }
+
+// ChildInput returns the output for a step that precedes a Child step, so the
+// child inherits this instance's run binding.
+func (t Task) ChildInput(payload any) (any, error) {
+ encoded, err := json.Marshal(payload)
+ if err != nil {
+ return nil, err
+ }
+ return envelope{Binding: t.binding, Payload: encoded}, nil
+}
+
+// Results decodes a fan-out step's output slot by slot. Francis encodes a failed
+// slot as an object whose only key is "error", so successful outputs must never
+// have that shape.
+func Results[T any](t Task, step string) ([]Result[T], error) {
+ var slots []json.RawMessage
+ if err := t.DecodeOutput(step, &slots); err != nil {
+ return nil, err
+ }
+ results := make([]Result[T], len(slots))
+ for index, slot := range slots {
+ var failure map[string]json.RawMessage
+ if json.Unmarshal(slot, &failure) == nil && len(failure) == 1 {
+ var message string
+ if raw, ok := failure["error"]; ok && json.Unmarshal(raw, &message) == nil {
+ results[index].Err = strings.TrimPrefix(message, actor.ErrJobPermanentFailure.Error()+": ")
+ continue
+ }
+ }
+ if err := json.Unmarshal(slot, &results[index].Value); err != nil {
+ return nil, fmt.Errorf("decode %s output %d: %w", step, index, err)
+ }
+ }
+ return results, nil
+}
+
+// truncate shortens a message to maxErrorLength bytes on a rune boundary.
+func truncate(message string) string {
+ if len(message) <= maxErrorLength {
+ return message
+ }
+ end := maxErrorLength
+ for end > 0 && !utf8.RuneStart(message[end]) {
+ end--
+ }
+ return message[:end] + "…"
+}
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index e72e81e42..a9062be2d 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -9,7 +9,6 @@ import (
"sync/atomic"
"time"
- "github.com/getarcaneapp/arcane/types/v2/containerregistry"
imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/api/types/events"
@@ -18,7 +17,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -34,11 +32,7 @@ const (
)
type imageUpdateScannerInternal interface {
- CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdatetypes.Response, error)
-}
-
-type registryCredentialLoaderInternal interface {
- GetEnabledRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error)
+ RunImageCheck(ctx context.Context, request imageupdatetypes.CheckRequest) (schedulertypes.Outcome, error)
}
type pollingSettingReaderInternal interface {
@@ -58,7 +52,6 @@ type projectImageRefsBackfillerInternal interface {
type ImageUpdateWatcher struct {
imageUpdateService imageUpdateScannerInternal
settingsService pollingSettingReaderInternal
- environmentService registryCredentialLoaderInternal
dockerService dockerEventBusProviderInternal
projectService projectImageRefsBackfillerInternal
dispatchMu sync.RWMutex
@@ -89,12 +82,11 @@ type ImageUpdateWatcher struct {
func NewImageUpdateWatcher(cfg *config.Config,
imageUpdateService *imageupdate.ImageUpdateService,
settingsService *settings.SettingsService,
- environmentService *environment.EnvironmentService,
dockerService *docker.DockerClientService,
projectService *project.ProjectService) (*ImageUpdateWatcher,
error,
) {
- if imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {
+ if imageUpdateService == nil || settingsService == nil || dockerService == nil || projectService == nil {
return nil, errors.New("image update watcher dependencies unavailable")
}
location := time.UTC
@@ -104,7 +96,6 @@ func NewImageUpdateWatcher(cfg *config.Config,
return &ImageUpdateWatcher{
imageUpdateService: imageUpdateService,
settingsService: settingsService,
- environmentService: environmentService,
dockerService: dockerService,
projectService: projectService,
location: location,
@@ -396,36 +387,13 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
}
slog.InfoContext(ctx, "image scan run started")
- creds, err := w.environmentService.GetEnabledRegistryCredentials(ctx)
- if err != nil {
- slog.WarnContext(ctx, "failed to load registry credentials for image scan", "error", err.Error())
- creds = nil
- }
-
- results, err := w.imageUpdateService.CheckAllImages(ctx, 0, creds)
+ outcome, err := w.imageUpdateService.RunImageCheck(ctx, imageupdatetypes.CheckRequest{All: true})
if err != nil {
return fmt.Errorf("image scan failed: %w", err)
}
-
- updates := 0
- scanErrors := 0
- targets := make([]schedulertypes.TargetOutcome, 0, len(results))
- for imageID, result := range results {
- if result == nil {
- continue
- }
- if result.Error != "" {
- scanErrors++
- targets = append(targets, schedulertypes.TargetOutcome{ID: imageID, Status: schedulertypes.Failed, Message: result.Error})
- continue
- }
- if result.HasUpdate {
- updates++
- }
- }
- slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)
- if scanErrors > 0 {
- return &schedulertypes.OutcomeError{Outcome: schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some image update checks failed", Targets: targets}}
+ slog.InfoContext(ctx, "image scan run completed", "status", outcome.Status, "message", outcome.Message)
+ if outcome.Status != schedulertypes.Succeeded {
+ return &schedulertypes.OutcomeError{Outcome: outcome}
}
return nil
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index afd665389..2b27b58f4 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -11,7 +11,6 @@ import (
"testing/synctest"
"time"
- "github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/api/types/events"
@@ -33,7 +32,7 @@ type imageUpdateScannerFakeInternal struct {
releaseCh <-chan struct{}
}
-func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ int, _ []containerregistry.Credential) (map[string]*imageupdate.Response, error) {
+func (s *imageUpdateScannerFakeInternal) RunImageCheck(ctx context.Context, _ imageupdate.CheckRequest) (schedulertypes.Outcome, error) {
s.mu.Lock()
s.calls++
call := s.calls
@@ -70,7 +69,7 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ i
panic("deliberate image scan panic")
}
- return map[string]*imageupdate.Response{}, err
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, err
}
func (s *imageUpdateScannerFakeInternal) countInternal() int {
@@ -120,12 +119,6 @@ func (s *pollingSettingReaderFakeInternal) setEnabledInternal(enabled bool) {
s.mu.Unlock()
}
-type registryCredentialLoaderFakeInternal struct{}
-
-func (registryCredentialLoaderFakeInternal) GetEnabledRegistryCredentials(context.Context) ([]containerregistry.Credential, error) {
- return nil, nil
-}
-
type dockerEventBusProviderFakeInternal struct {
eventBus *bus.DockerEventBus
}
@@ -197,7 +190,6 @@ func newImageUpdateWatcherForTestInternal(t *testing.T,
watcher := &ImageUpdateWatcher{
imageUpdateService: scanner,
settingsService: settings,
- environmentService: registryCredentialLoaderFakeInternal{},
dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},
projectService: backfiller,
trigger: make(chan struct{}, 1),
diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.go
index 38fc7bb86..e28c50510 100644
--- a/backend/pkg/scheduler/jobcontext/context.go
+++ b/backend/pkg/scheduler/jobcontext/context.go
@@ -3,6 +3,7 @@ package jobcontext
import (
"context"
+ "slices"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
@@ -12,6 +13,7 @@ type (
runKeyInternal struct{}
)
+// WithExecution binds the run a job executes and, when persist is set, how its target progress is saved.
func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(schedulertypes.TargetOutcome) error) context.Context {
ctx = context.WithValue(ctx, runKeyInternal{}, run)
if persist != nil {
@@ -20,6 +22,7 @@ func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(sch
return ctx
}
+// Progress saves target progress for the executing run; it is a no-op outside one.
func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {
if persist, ok := ctx.Value(progressKeyInternal{}).(func(schedulertypes.TargetOutcome) error); ok {
return persist(target)
@@ -27,6 +30,20 @@ func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {
return nil
}
+// Merge records target progress in place of the entry with its ID, keeping recovery data the update omits.
+func Merge(targets []schedulertypes.TargetOutcome, target schedulertypes.TargetOutcome) []schedulertypes.TargetOutcome {
+ index := slices.IndexFunc(targets, func(existing schedulertypes.TargetOutcome) bool { return existing.ID == target.ID })
+ if index < 0 {
+ return append(targets, target)
+ }
+ if len(target.RecoveryData) == 0 {
+ target.RecoveryData = targets[index].RecoveryData
+ }
+ targets[index] = target
+ return targets
+}
+
+// Run returns the run being executed, as of when its execution started.
func Run(ctx context.Context) (schedulertypes.Run, bool) {
run, ok := ctx.Value(runKeyInternal{}).(schedulertypes.Run)
return run, ok
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index f4b63ac40..65ee97444 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"log/slog"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -20,6 +21,8 @@ import (
kit "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
@@ -367,6 +370,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou
last := &current.Attempts[len(current.Attempts)-1]
last.FinishedAt = &now
last.Outcome = outcome
+ last.Outcome.Steps = nil
}
return nil
})
@@ -508,9 +512,53 @@ func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {
}
}
outcome.ActivityID = cmp.Or(outcome.ActivityID, previous.ActivityID)
+ if len(outcome.Steps) == 0 {
+ outcome.Steps = previous.Steps
+ }
return outcome
}
+// ExecutionContext binds durable progress for work executing a run. An empty
+// instanceID fences writes by run owner; otherwise by the active workflow instance.
+func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instanceID string) context.Context {
+ ctx = utils.WithActivityBatchID(ctx, run.ID)
+ return jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {
+ return q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if !fencedInternal(current, run, instanceID) {
+ return ErrRunConflict
+ }
+ current.Outcome.Targets = jobcontext.Merge(current.Outcome.Targets, target)
+ return nil
+ })
+ })
+}
+
+// RecordSteps projects workflow step progress onto a running run.
+func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID string, steps []st.StepOutcome) error {
+ return q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if !fencedInternal(current, run, instanceID) {
+ return ErrRunConflict
+ }
+ current.Outcome.Steps = steps
+ return nil
+ })
+}
+
+// fencedInternal reports whether work may still write to a run.
+func fencedInternal(current *st.Run, run st.Run, instanceID string) bool {
+ if instanceID == "" {
+ return current.Status == st.Running && current.Owner == run.Owner
+ }
+ return ActiveWorkflow(*current, instanceID)
+}
+
+// ActiveWorkflow reports whether a running run still follows a workflow instance.
+func ActiveWorkflow(run st.Run, instanceID string) bool {
+ return run.Status == st.Running && slices.ContainsFunc(run.Outcome.Targets, func(target st.TargetOutcome) bool {
+ return target.ResourceType == st.WorkflowTarget && target.ID == instanceID && !target.Status.Terminal()
+ })
+}
+
func (q *Coordinator) Active() bool { return q.enabled.Load() }
// Resolve releases legacy reviewed work without discarding execution evidence.
diff --git a/backend/pkg/scheduler/schedule/cron.go b/backend/pkg/scheduler/schedule/cron.go
index b074bef1d..d62afb519 100644
--- a/backend/pkg/scheduler/schedule/cron.go
+++ b/backend/pkg/scheduler/schedule/cron.go
@@ -2,7 +2,10 @@
package schedule
import (
+ "cmp"
+ "context"
"fmt"
+ "log/slog"
"strings"
"github.com/robfig/cron/v3"
@@ -25,6 +28,16 @@ func NormalizeSixField(value, subject string) (string, error) {
return normalized, nil
}
+// Or returns spec when it parses and fallback otherwise; job names the schedule in the warning.
+func Or(ctx context.Context, spec, fallback, job string) string {
+ spec = cmp.Or(spec, fallback)
+ if _, err := Parser().Parse(spec); err != nil {
+ slog.WarnContext(ctx, "Invalid cron expression, using default", "job", job, "invalidSchedule", spec, "error", err)
+ return fallback
+ }
+ return spec
+}
+
// Parser is shared by schedule validation, execution, and next-run display.
func Parser() cron.Parser {
return cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
diff --git a/backend/pkg/scheduler/scheduled_prune_job.go b/backend/pkg/scheduler/scheduled_prune_job.go
index 6d8e65035..a97477542 100644
--- a/backend/pkg/scheduler/scheduled_prune_job.go
+++ b/backend/pkg/scheduler/scheduled_prune_job.go
@@ -1,7 +1,6 @@
package scheduler
import (
- "cmp"
"context"
"log/slog"
@@ -39,15 +38,7 @@ func (j *ScheduledPruneJob) ShouldSchedule(ctx context.Context) bool {
}
func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {
- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"), "0 0 0 * * *")
-
- parser := scheduleutil.Parser()
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for scheduled-prune, using default", "invalidSchedule", schedule, "error", err)
- return "0 0 0 * * *"
- }
-
- return schedule
+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"), "0 0 0 * * *", "scheduled-prune")
}
func (j *ScheduledPruneJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
diff --git a/backend/pkg/scheduler/vulnerability_feature_test.go b/backend/pkg/scheduler/vulnerability_feature_test.go
deleted file mode 100644
index 674c7b962..000000000
--- a/backend/pkg/scheduler/vulnerability_feature_test.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package scheduler
-
-import (
- "testing"
-
- "github.com/getarcaneapp/arcane/types/v2/features"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/stretchr/testify/require"
-)
-
-func TestVulnerabilityFeatureGatesScheduledScansAndPatches(t *testing.T) {
- tests := []struct {
- name string
- scheduleEnabled bool
- featureEnabled bool
- reenable bool
- wantScheduled bool
- }{
- {name: "schedules disabled", featureEnabled: true},
- {name: "schedules enabled", scheduleEnabled: true, featureEnabled: true, wantScheduled: true},
- {name: "feature disabled", scheduleEnabled: true},
- {name: "both disabled"},
- {name: "feature re-enabled", scheduleEnabled: true, reenable: true, wantScheduled: true},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- settingsSvc := setupScheduledPruneSettingsServiceInternal(t)
- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "vulnerabilityScanEnabled", tt.scheduleEnabled))
- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "imageAutoPatchEnabled", tt.scheduleEnabled))
- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, tt.featureEnabled))
- if tt.reenable {
- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, true))
- }
- jobs := []interface {
- schedulertypes.Job
- schedulertypes.ConditionalJob
- }{NewVulnerabilityScanJob(nil, settingsSvc), NewAutoPatchJob(nil, settingsSvc)}
- for _, job := range jobs {
- t.Run(job.Name(), func(t *testing.T) {
- require.Equal(t, tt.wantScheduled, job.ShouldSchedule(t.Context()))
- if !tt.wantScheduled {
- outcome, err := job.Run(t.Context())
- require.NoError(t, err)
- require.Equal(t, schedulertypes.Skipped, outcome.Status)
- }
- })
- }
- })
- }
-}
diff --git a/backend/pkg/scheduler/vulnerability_scan_job.go b/backend/pkg/scheduler/vulnerability_scan_job.go
index aa54f0f3f..b4a5c31ab 100644
--- a/backend/pkg/scheduler/vulnerability_scan_job.go
+++ b/backend/pkg/scheduler/vulnerability_scan_job.go
@@ -1,99 +1,33 @@
package scheduler
import (
- "cmp"
"context"
- "errors"
- "log/slog"
- "github.com/getarcaneapp/arcane/types/v2"
"github.com/getarcaneapp/arcane/types/v2/features"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/getarcaneapp/arcane/types/v2/user"
- "go.getarcane.app/kit/pkg"
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
)
-const VulnerabilityScanJobName = "vulnerability-scan"
-
-// systemUser is used as the actor for scheduled vulnerability scans.
-var vulnerabilityScanSystemUser = user.Actor{
- Username: "System",
-}
-
-// VulnerabilityScanJob periodically scans all Docker images for vulnerabilities
-// using Trivy. It is opt-in via the "vulnerabilityScanEnabled" setting.
-type VulnerabilityScanJob struct {
- vulnerabilityService *vulnerability.VulnerabilityService
- settingsService *settings.SettingsService
-}
-
-// NewVulnerabilityScanJob creates a new VulnerabilityScanJob.
-func NewVulnerabilityScanJob(vulnerabilityService *vulnerability.VulnerabilityService, settingsService *settings.SettingsService) *VulnerabilityScanJob {
- return &VulnerabilityScanJob{
- vulnerabilityService: vulnerabilityService,
- settingsService: settingsService,
- }
-}
-
-func (j *VulnerabilityScanJob) Name() string {
- return VulnerabilityScanJobName
-}
-
-func (j *VulnerabilityScanJob) ShouldSchedule(ctx context.Context) bool {
- return j.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && j.settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)
-}
-
-// Schedule returns the cron expression for the job. Defaults to daily at midnight.
-func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {
- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"), "0 0 0 * * *")
-
- parser := scheduleutil.Parser()
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for vulnerability-scan, using default", "invalidSchedule", schedule, "error", err)
- return "0 0 0 * * *"
- }
-
- return schedule
-}
-
-func (j *VulnerabilityScanJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
- if !j.ShouldSchedule(ctx) {
- slog.DebugContext(ctx, "scheduled vulnerability scan disabled; skipping run")
- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
- }
-
- slog.InfoContext(ctx, "scheduled vulnerability scan started")
-
- // Remove scan records for images that no longer exist (fixes e.g. "5/3 images scanned").
- deleted, cleanupErr := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)
- if cleanupErr != nil {
- slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", cleanupErr)
- }
- if deleted > 0 {
- slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)
- }
-
- scanned, failed, err := j.vulnerabilityService.ScanAllImages(ctx, types.LocalDockerEnvironmentID, vulnerabilityScanSystemUser, false)
- if errors.Is(err, common.ErrFeatureDisabled) {
- status := kit.Ternary(scanned > 0, schedulertypes.Partial, schedulertypes.Skipped)
- return schedulertypes.Outcome{Status: status, Message: err.Error()}, nil
- }
- if err != nil {
- slog.ErrorContext(ctx, "scheduled vulnerability scan failed", "error", err)
- return schedulertypes.Outcome{}, err
- }
+const (
+ VulnerabilityScanJobName = "vulnerability-scan"
+ vulnerabilityScanDefaultCron = "0 0 0 * * *"
+)
- slog.InfoContext(ctx, "scheduled vulnerability scan completed",
- "scanned", scanned,
- "failed", failed,
- )
- if failed > 0 || cleanupErr != nil {
- return schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some scans or orphaned record cleanup failed"}, cleanupErr
+// NewVulnerabilityScanJob runs the durable vulnerability scan workflow. It is
+// opt-in via the "vulnerabilityScanEnabled" setting and defaults to daily at midnight.
+func NewVulnerabilityScanJob(engine *flow.Engine, vulnerabilityService *vulnerability.VulnerabilityService, settingsService *settings.SettingsService) *flow.Job {
+ return &flow.Job{
+ Engine: engine,
+ Workflow: vulnerabilityService.ScanWorkflow(),
+ JobName: VulnerabilityScanJobName,
+ ScheduleFn: func(ctx context.Context) string {
+ return scheduleutil.Or(ctx, settingsService.GetStringSetting(ctx, "vulnerabilityScanInterval", vulnerabilityScanDefaultCron), vulnerabilityScanDefaultCron, VulnerabilityScanJobName)
+ },
+ ShouldRunFn: func(ctx context.Context) bool {
+ return settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)
+ },
}
- return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
index 01b4c7762..e1e15bdfe 100644
--- a/backend/pkg/utils/francis/runtime.go
+++ b/backend/pkg/utils/francis/runtime.go
@@ -10,6 +10,7 @@ import (
"fmt"
"log/slog"
"net"
+ "slices"
"strconv"
"strings"
"sync"
@@ -17,6 +18,7 @@ import (
"time"
"github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/builtin/workflow"
"github.com/italypaleale/francis/components"
"github.com/italypaleale/francis/host/local"
"github.com/libtnb/sqlite"
@@ -34,6 +36,7 @@ type Runtime struct {
mu sync.Mutex
options []local.HostOption
registrations []registration
+ workflows []*workflow.Workflow
service *actor.Service
ready chan struct{}
done chan struct{}
@@ -147,6 +150,20 @@ func (r *Runtime) RegisterActor(actorType string, factory actor.Factory, options
return nil
}
+// RegisterWorkflow adds a Francis workflow, registered on every host attempt.
+func (r *Runtime) RegisterWorkflow(wf *workflow.Workflow) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.started {
+ return errors.New("workflows must be registered before startup")
+ }
+ if slices.ContainsFunc(r.workflows, func(existing *workflow.Workflow) bool { return existing.Name() == wf.Name() }) {
+ return fmt.Errorf("workflow %q is already registered", wf.Name())
+ }
+ r.workflows = append(r.workflows, wf)
+ return nil
+}
+
// Start waits for host registration. appCtx owns the running host; ctx only
// limits startup. onFailure must cancel the application without blocking.
func (r *Runtime) Start(ctx, appCtx context.Context, onFailure func(error)) error {
@@ -233,16 +250,26 @@ func (r *Runtime) runHost(ctx context.Context) (*local.Host, chan error, error)
if err != nil {
return nil, nil, err
}
+ var registerErr error
for _, registration := range r.registrations {
- if registerActorErr := host.RegisterActor(registration.actorType, registration.factory, registration.options...); registerActorErr != nil {
- // Running a canceled host closes its provider-owned connections.
- cleanupCtx, cancel := context.WithCancel(ctx)
- cancel()
- if cleanupErr := host.Run(cleanupCtx); cleanupErr != nil {
- return nil, nil, errors.Join(registerActorErr, fmt.Errorf("clean up unregistered actor host: %w", cleanupErr))
- }
- return nil, nil, registerActorErr
+ if registerErr = host.RegisterActor(registration.actorType, registration.factory, registration.options...); registerErr != nil {
+ break
+ }
+ }
+ for _, wf := range r.workflows {
+ if registerErr != nil {
+ break
+ }
+ registerErr = host.RegisterBuiltInActor(wf)
+ }
+ if registerErr != nil {
+ // Running a canceled host closes its provider-owned connections.
+ cleanupCtx, cancel := context.WithCancel(ctx)
+ cancel()
+ if cleanupErr := host.Run(cleanupCtx); cleanupErr != nil {
+ return nil, nil, errors.Join(registerErr, fmt.Errorf("clean up unregistered actor host: %w", cleanupErr))
}
+ return nil, nil, registerErr
}
localErrors := make(chan error, 1)
go func() { localErrors <- host.Run(ctx) }()
diff --git a/backend/pkg/utils/handlerutil/handlerutil.go b/backend/pkg/utils/handlerutil/handlerutil.go
index bc681ac38..3d86b33bc 100644
--- a/backend/pkg/utils/handlerutil/handlerutil.go
+++ b/backend/pkg/utils/handlerutil/handlerutil.go
@@ -30,6 +30,15 @@ type Out[T any] struct {
Body base.ApiResponse[T]
}
+// MessageOutput is a success response carrying a message and, when an activity tracks the work, its ID.
+func MessageOutput(message, activityID string) *Out[base.MessageResponse] {
+ data := base.MessageResponse{Message: message}
+ if activityID = strings.TrimSpace(activityID); activityID != "" {
+ data.ActivityID = &activityID
+ }
+ return &Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: data}}
+}
+
// Page is the huma response envelope for endpoints returning base.Paginated[T].
type Page[T any] struct {
Body base.Paginated[T]
diff --git a/cli/pkg/images/updates/cmd.go b/cli/pkg/images/updates/cmd.go
index 0c5c03aa9..062157566 100644
--- a/cli/pkg/images/updates/cmd.go
+++ b/cli/pkg/images/updates/cmd.go
@@ -26,6 +26,9 @@ var UpdatesCmd = &cobra.Command{
var checkAll bool
+// checkAllBodyInternal is an empty JSON object; older servers reject check-all without a body.
+var checkAllBodyInternal = struct{}{}
+
var checkCmd = &cobra.Command{
Use: "check [image-ref...]",
Short: "Check image references for updates",
@@ -44,7 +47,7 @@ var checkCmd = &cobra.Command{
return errors.New("--all cannot be combined with image references")
}
- result, checkAllUpdatesErr := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), imageupdate.CheckAllImagesRequest{})
+ result, checkAllUpdatesErr := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), checkAllBodyInternal)
if checkAllUpdatesErr != nil {
return fmt.Errorf("failed to check all updates: %w", checkAllUpdatesErr)
}
@@ -97,8 +100,7 @@ var checkAllCmd = &cobra.Command{
return err
}
- // The handler declares a non-pointer Body, so an empty body is a 400.
- result, err := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), imageupdate.CheckAllImagesRequest{})
+ result, err := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), checkAllBodyInternal)
if err != nil {
return fmt.Errorf("failed to check all updates: %w", err)
}
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index f170995ea..1f7463d76 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -3706,6 +3706,22 @@
"jobs_upgrade_required": "Upgrade this agent to use durable jobs.",
"jobs_offline_status": "This environment is offline. Showing its last confirmed job status.",
"jobs_run_targets": "Target outcomes",
+ "jobs_run_steps": "Steps",
+ "jobs_step_progress": "{completed}/{total}",
+ "jobs_step_failed": "{count} failed",
+ "jobs_step_prepare": "Prepare check",
+ "jobs_step_discover": "Discover images",
+ "jobs_step_check": "Check images",
+ "jobs_step_scan": "Scan images",
+ "jobs_step_patch": "Patch images",
+ "jobs_step_sync": "Sync from Git",
+ "jobs_step_backup": "Back up",
+ "jobs_step_tags": "Check tag policies",
+ "jobs_step_finalize": "Finalize",
+ "jobs_step_resume": "Check for saved plan",
+ "jobs_step_candidates": "Select candidates",
+ "jobs_step_plan": "Plan updates",
+ "jobs_step_apply": "Apply updates",
"jobs_worker_health": "Worker health",
"jobs_run_previous": "Previous page",
"jobs_run_next": "Next page",
diff --git a/frontend/src/lib/components/activity/activity-detail-panel.svelte b/frontend/src/lib/components/activity/activity-detail-panel.svelte
index ab15065a2..57169ed39 100644
--- a/frontend/src/lib/components/activity/activity-detail-panel.svelte
+++ b/frontend/src/lib/components/activity/activity-detail-panel.svelte
@@ -42,7 +42,7 @@
<Dialog.Description>{jobId}</Dialog.Description>
</Dialog.Header>
{#if historyOpen}
- <JobRunHistory {jobId} {environmentId} initialRunId={runId} />
+ <JobRunHistory {jobId} {environmentId} initialRunId={runId} onOpenActivity={() => (historyOpen = false)} />
{/if}
</Dialog.Content>
</Dialog.Root>
diff --git a/frontend/src/lib/components/job-card/job-detail-panel.svelte b/frontend/src/lib/components/job-card/job-detail-panel.svelte
index bf73c2d41..a2ab86735 100644
--- a/frontend/src/lib/components/job-card/job-detail-panel.svelte
+++ b/frontend/src/lib/components/job-card/job-detail-panel.svelte
@@ -157,7 +157,12 @@
</div>
{/if}
{#key activeHistoryJobId}
- <JobRunHistory jobId={activeHistoryJobId} {environmentId} onUpdate={onScheduleUpdate} />
+ <JobRunHistory
+ jobId={activeHistoryJobId}
+ {environmentId}
+ onUpdate={onScheduleUpdate}
+ onOpenActivity={() => (open = false)}
+ />
{/key}
{:else}
{#each errors as text (text)}
diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelte
index 79ec8098e..f6792ffa0 100644
--- a/frontend/src/lib/components/job-card/job-run-history.svelte
+++ b/frontend/src/lib/components/job-card/job-run-history.svelte
@@ -13,15 +13,28 @@
import { extractApiErrorMessage } from '#lib/utils/api.js';
import { formatDateTimeShort } from '#lib/utils/formatting.js';
- import { jobStatusLabel, jobStatusTone } from './job-status';
+ import { jobStatusLabel, jobStatusTone, jobStepLabel } from './job-status';
// Renders the run list and selected-run detail; the host decides where it lives (panel or dialog).
let {
jobId,
environmentId,
initialRunId,
- onUpdate
- }: { jobId: string; environmentId: string; initialRunId?: string; onUpdate?: () => void } = $props();
+ onUpdate,
+ onOpenActivity
+ }: {
+ jobId: string;
+ environmentId: string;
+ initialRunId?: string;
+ onUpdate?: () => void;
+ /** Lets the hosting dialog close so the Activity Center is not opened behind it. */
+ onOpenActivity?: () => void;
+ } = $props();
+
+ function openActivity(activityId: string, activityEnvironmentId: string) {
+ onOpenActivity?.();
+ activityStore.openCenter(activityId, undefined, activityEnvironmentId);
+ }
let page = $state(1);
// Hosts re-key this component per job, so the initial run only seeds state once.
// svelte-ignore state_referenced_locally
@@ -48,6 +61,8 @@
}
}));
const selectedRun = $derived(selected ? detail.data : undefined);
+ // The workflow target only links a run to its workflow instance.
+ const targets = $derived(selectedRun?.outcome.targets?.filter((target) => target.resourceType !== 'workflow') ?? []);
const hasPrev = $derived(page > 1);
const hasNext = $derived(!!runs.data && page * runs.data.limit < runs.data.total);
const metadata = $derived(
@@ -111,10 +126,8 @@
label = m.jobs_activity_output(),
activityEnvironmentId = environmentId
)}
- {#if activityId}<Button
- variant="link"
- size="inline"
- onclick={() => activityStore.openCenter(activityId, undefined, activityEnvironmentId)}>{label}</Button
+ {#if activityId}<Button variant="link" size="inline" onclick={() => openActivity(activityId, activityEnvironmentId)}
+ >{label}</Button
>{/if}
{/snippet}
@@ -181,9 +194,27 @@
{@render outcomeMessage(attempt.outcome.message)}
</div>
{/each}
- {#if run.outcome.targets?.length}
+ {#if run.outcome.steps?.length}
+ <h4 class="font-medium">{m.jobs_run_steps()}</h4>
+ <ol class="divide-y divide-border/50 rounded-lg border border-border/60">
+ {#each run.outcome.steps as step (step.name)}
+ <li class="space-y-1 px-2 py-1.5">
+ <div class="flex flex-wrap items-center justify-between gap-2">
+ <span>{jobStepLabel(step.name)}</span>
+ <span class="flex items-center gap-2 text-xs text-muted-foreground tabular-nums">
+ {#if step.total}{m.jobs_step_progress({ completed: step.completed ?? 0, total: step.total })}{/if}
+ {#if step.failed}<span class="text-destructive">{m.jobs_step_failed({ count: step.failed })}</span>{/if}
+ <Badge variant={jobStatusTone(step.status)} size="sm">{jobStatusLabel(step.status)}</Badge>
+ </span>
+ </div>
+ {@render outcomeMessage(step.message)}
+ </li>
+ {/each}
+ </ol>
+ {/if}
+ {#if targets.length}
<h4 class="font-medium">{m.jobs_run_targets()}</h4>
- {#each run.outcome.targets as target (target.id)}
+ {#each targets as target (target.id)}
<div class="space-y-1 rounded-lg border border-border/60 p-2">
<p class="break-all">{target.id} · {jobStatusLabel(target.status)}</p>
{@render outcomeMessage(target.message)}
diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.ts
index 1b7c3527f..1f37847f6 100644
--- a/frontend/src/lib/components/job-card/job-status.ts
+++ b/frontend/src/lib/components/job-card/job-status.ts
@@ -20,6 +20,26 @@ export function jobStatusLabel(status: string, isWorker = false): string {
return labels[status]?.() ?? m.jobs_status_unknown();
}
+/** Label for a workflow step, falling back to its name. */
+export function jobStepLabel(name: string): string {
+ const labels: Record<string, () => string> = {
+ prepare: m.jobs_step_prepare,
+ discover: m.jobs_step_discover,
+ check: m.jobs_step_check,
+ scan: m.jobs_step_scan,
+ patch: m.jobs_step_patch,
+ sync: m.jobs_step_sync,
+ backup: m.jobs_step_backup,
+ tags: m.jobs_step_tags,
+ finalize: m.jobs_step_finalize,
+ resume: m.jobs_step_resume,
+ candidates: m.jobs_step_candidates,
+ plan: m.jobs_step_plan,
+ apply: m.jobs_step_apply
+ };
+ return labels[name]?.() ?? name;
+}
+
export function jobNameLabel(job: JobStatus): string {
const [group = '', ...target] = job.id.split(':');
const labels: Record<string, () => string> = {
diff --git a/frontend/src/lib/types/job.ts b/frontend/src/lib/types/job.ts
index 8016367f0..ca0925684 100644
--- a/frontend/src/lib/types/job.ts
+++ b/frontend/src/lib/types/job.ts
@@ -9,8 +9,25 @@ export type JobRunStatus =
| 'failed'
| 'needs_attention'
| 'canceled';
-export type JobTargetOutcome = { id: string; status: JobRunStatus; message?: string; activityId?: string };
-export type JobOutcome = { status: JobRunStatus; message?: string; activityId?: string; targets?: JobTargetOutcome[] };
+export type JobTargetOutcome = { id: string; resourceType?: string; status: JobRunStatus; message?: string; activityId?: string };
+export type JobStepOutcome = {
+ name: string;
+ status: JobRunStatus;
+ total?: number;
+ completed?: number;
+ failed?: number;
+ attempts?: number;
+ message?: string;
+ startedAt?: string;
+ finishedAt?: string;
+};
+export type JobOutcome = {
+ status: JobRunStatus;
+ message?: string;
+ activityId?: string;
+ targets?: JobTargetOutcome[];
+ steps?: JobStepOutcome[];
+};
export type JobAttempt = { number: number; startedAt: string; finishedAt?: string; outcome: JobOutcome };
export type JobRun = {
id: string;
diff --git a/types/backup/durable_run.go b/types/backup/durable_run.go
deleted file mode 100644
index 755a04a87..000000000
--- a/types/backup/durable_run.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package backup
-
-import "github.com/getarcaneapp/arcane/types/v2/scheduler"
-
-// DurableRunCommand freezes the input accepted by a backup domain.
-type DurableRunCommand struct {
- UserID string `msgpack:"userID"`
- EnvironmentID string `msgpack:"environmentID"`
- Permission string `msgpack:"permission"`
- RequestedWithKey string `msgpack:"requestedWithKey"`
- Kind string `msgpack:"kind"`
- RunID string `msgpack:"runID"`
- ActivityID string `msgpack:"activityID"`
- Payload []byte `msgpack:"payload"`
-}
-
-// DurableRunState records dispatch and effect evidence across host restarts.
-type DurableRunState struct {
- Command DurableRunCommand `msgpack:"command"`
- Started bool `msgpack:"started"`
- Status scheduler.RunStatus `msgpack:"status"`
- Targets []scheduler.TargetOutcome `msgpack:"targets"`
- Error string `msgpack:"error"`
-}
diff --git a/types/backup/requester.go b/types/backup/requester.go
new file mode 100644
index 000000000..fa525fe5d
--- /dev/null
+++ b/types/backup/requester.go
@@ -0,0 +1,9 @@
+package backup
+
+// Requester is who started a manual backup; its workflow re-checks their permission on every delivery.
+type Requester struct {
+ UserID string `json:"userId"`
+ APIKeyID string `json:"apiKeyId,omitempty"`
+ EnvironmentID string `json:"environmentId"`
+ Permission string `json:"permission"`
+}
diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.go
index 20cb2a54c..660874474 100644
--- a/types/backup/system_volume_backup.go
+++ b/types/backup/system_volume_backup.go
@@ -104,6 +104,7 @@ type HistoryEntry struct {
Format string `json:"format,omitempty"`
LocalSnapshotID string `json:"localSnapshotId,omitempty"`
RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`
+ RemoteInstanceID string `json:"remoteInstanceId,omitempty"`
S3DestinationID string `json:"s3DestinationId,omitempty"`
S3DestinationName string `json:"s3DestinationName,omitempty"`
PolicyID string `json:"policyId,omitempty"`
diff --git a/types/imageupdate/image_update.go b/types/imageupdate/image_update.go
index 348d3f1ff..0c68d68ea 100644
--- a/types/imageupdate/image_update.go
+++ b/types/imageupdate/image_update.go
@@ -126,4 +126,11 @@ type CheckAllImagesRequest struct {
Credentials []containerregistry.Credential `json:"credentials,omitempty"`
}
+// CheckRequest selects the images an update check covers. All discovers every
+// local image; otherwise only ImageRefs are checked, and an empty list checks nothing.
+type CheckRequest struct {
+ ImageRefs []string `json:"imageRefs,omitempty"`
+ All bool `json:"all,omitempty"`
+}
+
type BatchResponse map[string]*Response
diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.go
index 6e0569af8..93070107c 100644
--- a/types/meta/job_metadata.go
+++ b/types/meta/job_metadata.go
@@ -89,13 +89,7 @@ var jobMetadataRegistry = map[string]JobMetadata{
ManagerOnly: false,
IsContinuous: false,
CanRunManually: true,
- Prerequisites: []JobPrerequisiteMetadata{
- {
- SettingKey: "analyticsEnabled",
- Label: "Analytics enabled",
- SettingsURL: "/settings/general",
- },
- },
+ Prerequisites: []JobPrerequisiteMetadata{},
},
"auto-update": {
ID: "auto-update",
diff --git a/types/scheduler/run.go b/types/scheduler/run.go
index 7ec541ed5..d969cec17 100644
--- a/types/scheduler/run.go
+++ b/types/scheduler/run.go
@@ -23,6 +23,9 @@ const (
Canceled RunStatus = "canceled"
)
+// WorkflowTarget marks the target that records a run's workflow instance.
+const WorkflowTarget = "workflow"
+
func (s RunStatus) Terminal() bool {
return s == Succeeded || s == Partial || s == Skipped || s == Failed || s == Canceled
}
@@ -36,11 +39,25 @@ type TargetOutcome struct {
ActivityID string `json:"activityId,omitempty"`
}
+// StepOutcome is one workflow step as projected onto its run.
+type StepOutcome struct {
+ Name string `json:"name"`
+ Status RunStatus `json:"status"`
+ Total int `json:"total,omitempty"`
+ Completed int `json:"completed,omitempty"`
+ Failed int `json:"failed,omitempty"`
+ Attempts int `json:"attempts,omitempty"`
+ Message string `json:"message,omitempty"`
+ StartedAt *time.Time `json:"startedAt,omitempty"`
+ FinishedAt *time.Time `json:"finishedAt,omitempty"`
+}
+
type Outcome struct {
Status RunStatus `json:"status"`
Message string `json:"message,omitempty"`
ActivityID string `json:"activityId,omitempty"`
Targets []TargetOutcome `json:"targets,omitempty"`
+ Steps []StepOutcome `json:"steps,omitempty"`
}
type Request struct {
diff --git a/types/updater/durable.go b/types/updater/durable.go
index 952db96b4..ce2fbca7b 100644
--- a/types/updater/durable.go
+++ b/types/updater/durable.go
@@ -1,21 +1,5 @@
package updater
-// SingleUpdateCommand contains the persisted inputs for an asynchronous update.
-type SingleUpdateCommand struct {
- ContainerID string
- ActivityID string
- UserID string
- KeyID string
-}
-
-type SingleUpdateState struct {
- Command SingleUpdateCommand
- Status string
- Target *FrozenUpdateTarget
- Result *Result
- Failure string
-}
-
// FrozenUpdateTarget identifies the selected container and immutable image intent.
type FrozenUpdateTarget struct {
ContainerID string
diff --git a/types/volume/backup.go b/types/volume/backup.go
index 016cc04c1..b23b3412a 100644
--- a/types/volume/backup.go
+++ b/types/volume/backup.go
@@ -81,6 +81,29 @@ type BackupEntry struct {
Type backup.ManagementType `json:"type" doc:"Whether the backup was system-managed or volume-managed"`
}
+// Entry is the backup as the API lists it.
+func (b *Backup) Entry() BackupEntry {
+ return BackupEntry{
+ RemoteAvailable: b.RemoteAvailable,
+ ActivityID: b.ActivityID,
+ ID: b.ID,
+ VolumeName: b.VolumeName,
+ Size: b.Size,
+ CreatedAt: b.CreatedAt.Format(time.RFC3339),
+ Status: b.Status,
+ Trigger: b.Trigger,
+ Destination: b.Destination,
+ Format: b.Format,
+ LocalSnapshotID: b.LocalSnapshotID,
+ RemoteSnapshotID: b.RemoteSnapshotID,
+ S3DestinationID: b.S3DestinationID,
+ S3DestinationName: b.S3DestinationName,
+ PolicyID: b.PolicyID,
+ Error: b.Error,
+ Type: b.Type,
+ }
+}
+
type BackupPolicy struct {
ID string `json:"id"`
VolumeName string `json:"volumeName"`