Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: self updater not restarting container properly (#2897)
Kyle Mendell7706fe2ef8be
diff --git a/backend/api/handlers/system.go b/backend/api/handlers/system.goindex d76afd569..389c0604d 100644--- a/backend/api/handlers/system.go+++ b/backend/api/handlers/system.go@@ -20,6 +20,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/system"dockersystem "github.com/moby/moby/api/types/system""github.com/moby/moby/client"+ updatertypes "go.getarcane.app/updater/types")// SystemHandler handles system management endpoints.@@ -518,7 +519,7 @@ func (h *SystemHandler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradslog.Info("System upgrade triggered", "user", user.Username, "userId", user.ID)- err = h.upgradeService.TriggerUpgradeViaCLI(ctx, *user)+ err = h.upgradeService.TriggerUpgradeViaCLI(ctx, *user, updatertypes.SelfUpdateTarget{})if err != nil {slog.Error("System upgrade failed", "error", err, "user", user.Username)diff --git a/backend/go.mod b/backend/go.modindex 9f15bdd0b..9ab55fbd1 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -55,7 +55,7 @@ require (github.com/spf13/cobra v1.10.2github.com/stretchr/testify v1.11.1github.com/wneessen/go-mail v0.7.3- go.getarcane.app/updater v0.3.7+ go.getarcane.app/updater v0.3.9golang.org/x/crypto v0.52.0golang.org/x/mod v0.36.0golang.org/x/net v0.55.0diff --git a/backend/go.sum b/backend/go.sumindex 0dd650543..c5a57464c 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -540,8 +540,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9degithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=-go.getarcane.app/updater v0.3.7 h1:WqHsxudvt8gvCR7iSHdxdzwH0+hM0FAh64bSypBX9Mo=-go.getarcane.app/updater v0.3.7/go.mod h1:9RZOKaBbpUJfVA4Q9/jvqtjvWBwCnVw0OkPeQCnqAAo=+go.getarcane.app/updater v0.3.9 h1:+qVmC1ruV7tDVONXDQcb56seSlhshMEIvH877XxZA40=+go.getarcane.app/updater v0.3.9/go.mod h1:4+jvmGU+HdDLHTuqFZ/NJ7lzZzF0F2AVvqnap5uJQaQ=go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex e8040b5fe..90d39d371 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -24,6 +24,13 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul} else if failed > 0 {slog.InfoContext(appCtx, "Marked stale image update checks as failed", "count", failed)}++ resolved, err := appServices.Activity.ResolveStaleAutoUpdateActivities(appCtx)+ if err != nil {+ slog.WarnContext(appCtx, "Failed to resolve stale auto-update activities", "count", resolved, "error", err)+ } else if resolved > 0 {+ slog.InfoContext(appCtx, "Resolved stale auto-update activities", "count", resolved)+ }}newScheduler.RegisterJob(jobs.AutoUpdate)diff --git a/backend/internal/services/activity_service.go b/backend/internal/services/activity_service.goindex 3876da666..c5a99c290 100644--- a/backend/internal/services/activity_service.go+++ b/backend/internal/services/activity_service.go@@ -491,6 +491,77 @@ func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64return failed, errors.Join(failErrs...)}+// PatchActivityMetadata merges patch into the activity's existing metadata,+// unlike UpdateActivity which replaces the metadata wholesale.+func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID string, patch models.JSON) error {+ if err := s.checkInitInternal(); err != nil {+ return err+ }+ activityID = strings.TrimSpace(activityID)+ if activityID == "" {+ return errors.New("activity id is required")+ }+ if len(patch) == 0 {+ return nil+ }++ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var activity models.Activity+ if err := tx.First(&activity, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load activity: %w", err)+ }+ merged := cloneJSONInternal(activity.Metadata)+ if merged == nil {+ merged = models.JSON{}+ }+ maps.Copy(merged, patch)+ if err := tx.Model(&models.Activity{}).Where("id = ?", activityID).+ Updates(map[string]any{"metadata": merged, "updated_at": time.Now()}).Error; err != nil {+ return fmt.Errorf("failed to patch activity metadata: %w", err)+ }+ return nil+ })+}++// ResolveStaleAutoUpdateActivities finalizes auto-update activities left+// running by a prior process lifetime. A run whose metadata marks a triggered+// self-update completed by restarting Arcane, so it is recorded as success;+// anything else still running at startup was interrupted and is failed.+func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context) (int64, error) {+ if s == nil || s.db == nil {+ return 0, nil+ }++ var stale []models.Activity+ if err := s.db.WithContext(ctx).+ Where("type = ? AND status = ?", models.ActivityTypeAutoUpdate, models.ActivityStatusRunning).+ Find(&stale).Error; err != nil {+ return 0, fmt.Errorf("find stale auto-update activities: %w", err)+ }++ var resolved int64+ var resolveErrs []error+ for i := range stale {+ status := models.ActivityStatusFailed+ message := "Auto-update interrupted by Arcane restart"+ var errMessage *string+ if selfUpdate, _ := stale[i].Metadata["selfUpdateTriggered"].(bool); selfUpdate {+ status = models.ActivityStatusSuccess+ message = "Auto-update completed — Arcane restarted with the updated image"+ } else {+ errText := message+ errMessage = &errText+ }+ if _, err := s.CompleteActivity(ctx, stale[i].ID, status, message, errMessage); err != nil {+ resolveErrs = append(resolveErrs, fmt.Errorf("resolve stale auto-update activity %s: %w", stale[i].ID, err))+ continue+ }+ resolved+++ }++ return resolved, errors.Join(resolveErrs...)+}+func completeActivityUpdatesInternal(startedAt time.Time, status models.ActivityStatus, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {updates := map[string]any{"status": status,diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.goindex 49d0843d7..de7a2ad7e 100644--- a/backend/internal/services/activity_service_test.go+++ b/backend/internal/services/activity_service_test.go@@ -417,6 +417,54 @@ func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {require.Equal(t, models.ActivityStatusSuccess, completed.Status)}+func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {+ ctx := context.Background()+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db)++ selfUpdateRun, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeAutoUpdate,+ LatestMessage: "updating",+ Metadata: models.JSON{"dryRun": false},+ })+ require.NoError(t, err)+ require.NoError(t, service.PatchActivityMetadata(ctx, selfUpdateRun.ID, models.JSON{"selfUpdateTriggered": true}))++ interruptedRun, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeAutoUpdate,+ LatestMessage: "updating",+ })+ require.NoError(t, err)+ otherType, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeImagePull,+ LatestMessage: "pulling",+ })+ require.NoError(t, err)++ resolved, err := service.ResolveStaleAutoUpdateActivities(ctx)+ require.NoError(t, err)+ require.EqualValues(t, 2, resolved)++ var selfUpdated models.Activity+ require.NoError(t, db.First(&selfUpdated, "id = ?", selfUpdateRun.ID).Error)+ require.Equal(t, models.ActivityStatusSuccess, selfUpdated.Status)+ require.NotNil(t, selfUpdated.EndedAt)+ require.Contains(t, selfUpdated.LatestMessage, "restarted with the updated image")+ require.Equal(t, false, selfUpdated.Metadata["dryRun"])++ var interrupted models.Activity+ require.NoError(t, db.First(&interrupted, "id = ?", interruptedRun.ID).Error)+ require.Equal(t, models.ActivityStatusFailed, interrupted.Status)+ require.Contains(t, interrupted.LatestMessage, "interrupted")++ var other models.Activity+ require.NoError(t, db.First(&other, "id = ?", otherType.ID).Error)+ require.Equal(t, models.ActivityStatusRunning, other.Status)+}+func receiveActivityEventInternal(t *testing.T, events <-chan activitytypes.StreamEvent) activitytypes.StreamEvent {t.Helper()diff --git a/backend/internal/services/system_upgrade_service.go b/backend/internal/services/system_upgrade_service.goindex 79bd6374a..33d13e1bb 100644--- a/backend/internal/services/system_upgrade_service.go+++ b/backend/internal/services/system_upgrade_service.go@@ -6,6 +6,7 @@ import ("fmt""log/slog""os"+ "slices""strings""sync/atomic""time"@@ -19,6 +20,7 @@ import (mounttypes "github.com/moby/moby/api/types/mount""github.com/moby/moby/client"libupdater "go.getarcane.app/updater/pkg/labels"+ updatertypes "go.getarcane.app/updater/types")const defaultArcaneUpgraderImageInternal = "ghcr.io/getarcaneapp/arcane:latest"@@ -75,17 +77,23 @@ func (s *SystemUpgradeService) CanUpgrade(ctx context.Context) (bool, error) {}// TriggerUpgradeViaCLI spawns the upgrade CLI command in a separate container-// This avoids self-termination issues by running the upgrade from outside-func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user models.User) error {+// This avoids self-termination issues by running the upgrade from outside.+// A zero-value target upgrades the current container to its own image tag;+// the updater engine passes an explicit target with the resolved new image.+func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user models.User, target updatertypes.SelfUpdateTarget) error {if !s.upgrading.CompareAndSwap(false, true) {return &common.UpgradeInProgressError{}}defer s.upgrading.Store(false)- // Get current container name- containerId, err := s.getCurrentContainerIDInternal()- if err != nil {- return fmt.Errorf("get current container: %w", err)+ containerId := strings.TrimSpace(target.ContainerID)+ if containerId == "" {+ // Fall back to the container this process runs in+ var err error+ containerId, err = s.getCurrentContainerIDInternal()+ if err != nil {+ return fmt.Errorf("get current container: %w", err)+ }}currentContainer, err := s.findArcaneContainerInternal(ctx, containerId)@@ -101,23 +109,30 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user mobinaryPath = determineUpgradeBinaryPathInternal(currentContainer.Config.Labels)}+ targetImage := strings.TrimSpace(target.NewImageRef)+// Log upgrade eventmetadata := models.JSON{"action": "system_upgrade_cli","containerId": containerId,"containerName": containerName,"method": "cli",+ "targetImage": targetImage,}if err := s.eventService.LogUserEvent(ctx, models.EventTypeSystemUpgrade, user.ID, user.Username, metadata); err != nil {slog.Warn("Failed to log upgrade event", "error", err)}- // Use the same image reference as the currently running Arcane container for the upgrader.- // This avoids mismatches where a newer/older upgrader CLI expects different behavior.- upgraderImage := defaultArcaneUpgraderImageInternal- if currentContainer.Config != nil {- if img := strings.TrimSpace(currentContainer.Config.Image); img != "" {- upgraderImage = img+ // Run the upgrader from the image we are upgrading to, so the upgrade CLI+ // is the new version. Without a resolved target image, fall back to the+ // running container's own image reference.+ upgraderImage := targetImage+ if upgraderImage == "" {+ upgraderImage = defaultArcaneUpgraderImageInternal+ if currentContainer.Config != nil {+ if img := strings.TrimSpace(currentContainer.Config.Image); img != "" {+ upgraderImage = img+ }}}slog.Debug("Using upgrader image", "image", upgraderImage)@@ -180,10 +195,19 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user moreturn fmt.Errorf("resolve upgrader docker runtime: %w", err)}+ upgradeCmd := []string{binaryPath, "upgrade", "--container", containerName}+ if targetImage != "" {+ upgradeCmd = append(upgradeCmd, "--image", targetImage)+ }+config := &containertypes.Config{Image: upgraderImage,- Cmd: []string{binaryPath, "upgrade", "--container", containerName},- Env: runtimeOptions.ContainerEnv,+ Cmd: upgradeCmd,+ // The upgrader needs root for the Docker socket; unlike the server it+ // is short-lived and never goes through the runtime-identity drop, so+ // don't rely on the image's default user.+ User: "0:0",+ Env: runtimeOptions.ContainerEnv,Labels: map[string]string{"com.getarcaneapp.arcane.upgrader": "true","com.getarcaneapp.arcane": "true",@@ -205,6 +229,19 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user moMounts: mounts,NetworkMode: runtimeOptions.NetworkMode,}+ // Inherit the security context that lets the running Arcane container reach+ // the Docker socket (e.g. SELinux label=disable, privileged); the upgrader+ // needs the same access on hardened hosts.+ if currentContainer.HostConfig != nil {+ hostConfig.SecurityOpt = slices.Clone(currentContainer.HostConfig.SecurityOpt)+ hostConfig.Privileged = currentContainer.HostConfig.Privileged+ }+ // On SELinux-enforcing hosts the socket carries container_var_run_t, which+ // container processes cannot connect to regardless of UID; without an+ // explicit label opt the upgrader would exit with EACCES and auto-remove.+ if !hostConfig.Privileged && !hasSELinuxLabelOptInternal(hostConfig.SecurityOpt) && daemonHasSELinuxEnabledInternal(ctx, dockerClient) {+ hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")+ }containerName = fmt.Sprintf("%s-upgrader-%d", containerName, time.Now().Unix())@@ -228,6 +265,26 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user moreturn nil}+// hasSELinuxLabelOptInternal reports whether the security options already set+// an SELinux label policy (e.g. "label=disable", "label:disable", "label=type:...").+func hasSELinuxLabelOptInternal(securityOpts []string) bool {+ for _, opt := range securityOpts {+ if strings.HasPrefix(strings.TrimSpace(opt), "label") {+ return true+ }+ }+ return false+}++func daemonHasSELinuxEnabledInternal(ctx context.Context, dockerClient *client.Client) bool {+ infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})+ if err != nil {+ slog.Debug("Failed to query daemon info for SELinux detection", "error", err)+ return false+ }+ return slices.Contains(infoResult.Info.SecurityOptions, "name=selinux")+}+func determineUpgradeBinaryPathInternal(labels map[string]string) string {if libupdater.IsArcaneAgentContainer(labels) {return "/app/arcane-agent"diff --git a/backend/internal/services/updater_service.go b/backend/internal/services/updater_service.goindex 14f820c7b..54318736b 100644--- a/backend/internal/services/updater_service.go+++ b/backend/internal/services/updater_service.go@@ -55,7 +55,7 @@ type updaterDependenciesInternal struct {}type selfUpgradeServiceInternal interface {- TriggerUpgradeViaCLI(ctx context.Context, user models.User) error+ TriggerUpgradeViaCLI(ctx context.Context, user models.User, target moduletypes.SelfUpdateTarget) error}// NewUpdaterService constructs the Arcane updater facade.@@ -107,10 +107,22 @@ func (s *UpdaterService) configInternal(ctx context.Context) moduleapi.Config {UsedImageCollector: moduleapi.UsedImageCollectorFunc(s.CollectUsedImages),LabelPolicy: labels.DefaultLabelPolicy(),AllowComposeStandaloneFallback: s.allowComposeStandaloneFallbackInternal(ctx),+ SelfContainerID: selfContainerIDInternal(),Logger: s.loggerInternal(),}}+// selfContainerIDInternal returns the ID of the container Arcane runs in, so+// the updater engine routes it through the CLI self-updater even when the+// container is missing the Arcane labels. Empty when not running in Docker.+func selfContainerIDInternal() string {+ id, err := dockerutil.GetCurrentContainerID()+ if err != nil {+ return ""+ }+ return id+}+func (s *UpdaterService) engineInternal() *moduleapi.Service {return s.engine}@@ -466,12 +478,35 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target moduletyp}return fmt.Errorf("%s self-update requires CLI upgrade service", instanceType)}- if err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser); err != nil {++ // A server self-update stops this process before the run can complete its+ // activity, so annotate the activity first; startup reconciliation uses+ // the metadata flag to finalize it after the restart.+ if target.InstanceType != "agent" {+ s.markSelfUpdateTriggeredInternal(ctx, target)+ }++ if err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {return fmt.Errorf("CLI upgrade failed: %w", err)}return nil}+func (s *UpdaterService) markSelfUpdateTriggeredInternal(ctx context.Context, target moduletypes.SelfUpdateTarget) {+ activityID := activityIDFromContextInternal(ctx)+ if s.deps.Activity == nil || activityID == "" {+ return+ }+ message := "Self-update initiated — Arcane will restart"+ if ref := strings.TrimSpace(target.NewImageRef); ref != "" {+ message = "Self-update initiated — Arcane will restart with " + ref+ }+ s.appendAutoUpdateActivityMessageInternal(ctx, activityID, message, "Self-update", 90)+ if err := s.deps.Activity.PatchActivityMetadata(ctx, activityID, models.JSON{"selfUpdateTriggered": true}); err != nil {+ slog.DebugContext(ctx, "failed to mark self-update on activity", "activityId", activityID, "error", err)+ }+}+// Notify sends Arcane's container update notification.func (s *UpdaterService) Notify(ctx context.Context, notification moduletypes.Notification) error {if s == nil || s.deps.Notifications == nil {diff --git a/backend/internal/services/updater_service_test.go b/backend/internal/services/updater_service_test.goindex 436c27d7c..faf50fa2f 100644--- a/backend/internal/services/updater_service_test.go+++ b/backend/internal/services/updater_service_test.go@@ -174,14 +174,16 @@ func newUpdaterApplyPendingDockerServerInternal(}type mockSystemUpgradeServiceInternal struct {- triggerCalled bool- triggerError error- capturedUser *models.User+ triggerCalled bool+ triggerError error+ capturedUser *models.User+ capturedTarget *moduletypes.SelfUpdateTarget}-func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user models.User) error {+func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user models.User, target moduletypes.SelfUpdateTarget) error {m.triggerCalled = truem.capturedUser = &user+ m.capturedTarget = &targetreturn m.triggerError}@@ -279,6 +281,9 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {require.NotNil(t, mockUpgrade.capturedUser)assert.Equal(t, systemUser.ID, mockUpgrade.capturedUser.ID)assert.Equal(t, systemUser.Username, mockUpgrade.capturedUser.Username)+ require.NotNil(t, mockUpgrade.capturedTarget)+ assert.Equal(t, "container-1", mockUpgrade.capturedTarget.ContainerID)+ assert.Equal(t, "arcane", mockUpgrade.capturedTarget.ContainerName)})t.Run("agent label triggers upgrade", func(t *testing.T) {diff --git a/go.work.sum b/go.work.sumindex 817c50b6b..6d8054541 100644--- a/go.work.sum+++ b/go.work.sum@@ -1161,6 +1161,8 @@ go.etcd.io/etcd/server/v3 v3.5.6 h1:RXuwaB8AMiV62TqcqIt4O4bG8NWjsxOkDJVT3MZI5Ds=go.etcd.io/etcd/server/v3 v3.5.6/go.mod h1:6/Gfe8XTGXQJgLYQ65oGKMfPivb2EASLUSMSWN9Sroo=go.getarcane.app/updater v0.3.6 h1:+BqoycaRsHj+cLVSzcYQuqbHxUKbtmaozMNblz5bVSw=go.getarcane.app/updater v0.3.6/go.mod h1:2ENitZR/lA5MuBn60OZYR5wvU3ZCE+pAgfrebfOlRDA=+go.getarcane.app/updater v0.3.9 h1:+qVmC1ruV7tDVONXDQcb56seSlhshMEIvH877XxZA40=+go.getarcane.app/updater v0.3.9/go.mod h1:4+jvmGU+HdDLHTuqFZ/NJ7lzZzF0F2AVvqnap5uJQaQ=go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI=go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=